44 static int sz2ord(size_t size)  in sz2ord()  argument
46 	return __builtin_ctzll(size / pagesize);  in sz2ord()
75 static bool range_is_swapped(void *addr, size_t size)  in range_is_swapped()  argument
77 	for (; size; addr += pagesize, size -= pagesize)  in range_is_swapped()
112 static int child_memcmp_fn(char *mem, size_t size,  in child_memcmp_fn()  argument
115 	char *old = malloc(size);  in child_memcmp_fn()
119 	memcpy(old, mem, size);  in child_memcmp_fn()
127 	return memcmp(old, mem, size);  in child_memcmp_fn()
130 static int child_vmsplice_memcmp_fn(char *mem, size_t size,  in child_vmsplice_memcmp_fn()  argument
135 		.iov_len = size,  in child_vmsplice_memcmp_fn()
142 	old = malloc(size);  in child_vmsplice_memcmp_fn()
143 	new = malloc(size);  in child_vmsplice_memcmp_fn()
146 	memcpy(old, mem, size);  in child_vmsplice_memcmp_fn()
159 	if (munmap(mem, size) < 0)  in child_vmsplice_memcmp_fn()
177 typedef int (*child_fn)(char *mem, size_t size, struct comm_pipes *comm_pipes);
179 static void do_test_cow_in_parent(char *mem, size_t size, bool do_mprotect,  in do_test_cow_in_parent()  argument
198 		exit(fn(mem, size, &comm_pipes));  in do_test_cow_in_parent()
209 		ret = mprotect(mem, size, PROT_READ);  in do_test_cow_in_parent()
218 		ret = mprotect(mem, size, PROT_READ|PROT_WRITE);  in do_test_cow_in_parent()
229 	memset(mem, 0xff, size);  in do_test_cow_in_parent()
256 static void test_cow_in_parent(char *mem, size_t size, bool is_hugetlb)  in test_cow_in_parent()  argument
258 	do_test_cow_in_parent(mem, size, false, child_memcmp_fn, false);  in test_cow_in_parent()
261 static void test_cow_in_parent_mprotect(char *mem, size_t size, bool is_hugetlb)  in test_cow_in_parent_mprotect()  argument
263 	do_test_cow_in_parent(mem, size, true, child_memcmp_fn, false);  in test_cow_in_parent_mprotect()
266 static void test_vmsplice_in_child(char *mem, size_t size, bool is_hugetlb)  in test_vmsplice_in_child()  argument
268 	do_test_cow_in_parent(mem, size, false, child_vmsplice_memcmp_fn,  in test_vmsplice_in_child()
272 static void test_vmsplice_in_child_mprotect(char *mem, size_t size,  in test_vmsplice_in_child_mprotect()  argument
275 	do_test_cow_in_parent(mem, size, true, child_vmsplice_memcmp_fn,  in test_vmsplice_in_child_mprotect()
279 static void do_test_vmsplice_in_parent(char *mem, size_t size,  in do_test_vmsplice_in_parent()  argument
284 		.iov_len = size,  in do_test_vmsplice_in_parent()
292 	old = malloc(size);  in do_test_vmsplice_in_parent()
293 	new = malloc(size);  in do_test_vmsplice_in_parent()
295 	memcpy(old, mem, size);  in do_test_vmsplice_in_parent()
328 		memset(mem, 0xff, size);  in do_test_vmsplice_in_parent()
344 	if (munmap(mem, size) < 0) {  in do_test_vmsplice_in_parent()
393 static void test_vmsplice_before_fork(char *mem, size_t size, bool is_hugetlb)  in test_vmsplice_before_fork()  argument
395 	do_test_vmsplice_in_parent(mem, size, true, is_hugetlb);  in test_vmsplice_before_fork()
398 static void test_vmsplice_after_fork(char *mem, size_t size, bool is_hugetlb)  in test_vmsplice_after_fork()  argument
400 	do_test_vmsplice_in_parent(mem, size, false, is_hugetlb);  in test_vmsplice_after_fork()
404 static void do_test_iouring(char *mem, size_t size, bool use_fork)  in do_test_iouring()  argument
431 	tmp = malloc(size);  in do_test_iouring()
454 	iov.iov_len = size;  in do_test_iouring()
489 		ret = mprotect(mem, size, PROT_READ);  in do_test_iouring()
497 		ret = mprotect(mem, size, PROT_READ | PROT_WRITE);  in do_test_iouring()
509 	memset(mem, 0xff, size);  in do_test_iouring()
516 	io_uring_prep_write_fixed(sqe, fd, mem, size, 0, 0);  in do_test_iouring()
532 	if (cqe->res != size) {  in do_test_iouring()
541 	while (total < size) {  in do_test_iouring()
542 		cur = pread(fd, tmp + total, size - total, total);  in do_test_iouring()
552 	if (!memcmp(mem, tmp, size)) {  in do_test_iouring()
576 static void test_iouring_ro(char *mem, size_t size, bool is_hugetlb)  in test_iouring_ro()  argument
578 	do_test_iouring(mem, size, false);  in test_iouring_ro()
581 static void test_iouring_fork(char *mem, size_t size, bool is_hugetlb)  in test_iouring_fork()  argument
583 	do_test_iouring(mem, size, true);  in test_iouring_fork()
595 static void do_test_ro_pin(char *mem, size_t size, enum ro_pin_test test,  in do_test_ro_pin()  argument
610 	tmp = malloc(size);  in do_test_ro_pin()
668 		ret = mprotect(mem, size, PROT_READ);  in do_test_ro_pin()
670 		ret |= mprotect(mem, size, PROT_READ | PROT_WRITE);  in do_test_ro_pin()
683 	args.size = size;  in do_test_ro_pin()
697 	memset(mem, 0xff, size);  in do_test_ro_pin()
709 		if (!memcmp(mem, tmp, size)) {  in do_test_ro_pin()
737 static void test_ro_pin_on_shared(char *mem, size_t size, bool is_hugetlb)  in test_ro_pin_on_shared()  argument
739 	do_test_ro_pin(mem, size, RO_PIN_TEST_SHARED, false);  in test_ro_pin_on_shared()
742 static void test_ro_fast_pin_on_shared(char *mem, size_t size, bool is_hugetlb)  in test_ro_fast_pin_on_shared()  argument
744 	do_test_ro_pin(mem, size, RO_PIN_TEST_SHARED, true);  in test_ro_fast_pin_on_shared()
747 static void test_ro_pin_on_ro_previously_shared(char *mem, size_t size,  in test_ro_pin_on_ro_previously_shared()  argument
750 	do_test_ro_pin(mem, size, RO_PIN_TEST_PREVIOUSLY_SHARED, false);  in test_ro_pin_on_ro_previously_shared()
753 static void test_ro_fast_pin_on_ro_previously_shared(char *mem, size_t size,  in test_ro_fast_pin_on_ro_previously_shared()  argument
756 	do_test_ro_pin(mem, size, RO_PIN_TEST_PREVIOUSLY_SHARED, true);  in test_ro_fast_pin_on_ro_previously_shared()
759 static void test_ro_pin_on_ro_exclusive(char *mem, size_t size,  in test_ro_pin_on_ro_exclusive()  argument
762 	do_test_ro_pin(mem, size, RO_PIN_TEST_RO_EXCLUSIVE, false);  in test_ro_pin_on_ro_exclusive()
765 static void test_ro_fast_pin_on_ro_exclusive(char *mem, size_t size,  in test_ro_fast_pin_on_ro_exclusive()  argument
768 	do_test_ro_pin(mem, size, RO_PIN_TEST_RO_EXCLUSIVE, true);  in test_ro_fast_pin_on_ro_exclusive()
771 typedef void (*test_fn)(char *mem, size_t size, bool hugetlb);
837 	size_t size, mmap_size, mremap_size;  in do_run_with_thp()  local
872 	size = thpsize;  in do_run_with_thp()
910 		size = pagesize;  in do_run_with_thp()
932 		size = mremap_size;  in do_run_with_thp()
971 		madvise(mem, size, MADV_PAGEOUT);  in do_run_with_thp()
972 		if (!range_is_swapped(mem, size)) {  in do_run_with_thp()
982 	fn(mem, size, false);  in do_run_with_thp()
989 static void run_with_thp(test_fn fn, const char *desc, size_t size)  in run_with_thp()  argument
992 		desc, size / 1024);  in run_with_thp()
993 	do_run_with_thp(fn, THP_RUN_PMD, size);  in run_with_thp()
996 static void run_with_thp_swap(test_fn fn, const char *desc, size_t size)  in run_with_thp_swap()  argument
999 		desc, size / 1024);  in run_with_thp_swap()
1000 	do_run_with_thp(fn, THP_RUN_PMD_SWAPOUT, size);  in run_with_thp_swap()
1003 static void run_with_pte_mapped_thp(test_fn fn, const char *desc, size_t size)  in run_with_pte_mapped_thp()  argument
1006 		desc, size / 1024);  in run_with_pte_mapped_thp()
1007 	do_run_with_thp(fn, THP_RUN_PTE, size);  in run_with_pte_mapped_thp()
1010 static void run_with_pte_mapped_thp_swap(test_fn fn, const char *desc, size_t size)  in run_with_pte_mapped_thp_swap()  argument
1013 		desc, size / 1024);  in run_with_pte_mapped_thp_swap()
1014 	do_run_with_thp(fn, THP_RUN_PTE_SWAPOUT, size);  in run_with_pte_mapped_thp_swap()
1017 static void run_with_single_pte_of_thp(test_fn fn, const char *desc, size_t size)  in run_with_single_pte_of_thp()  argument
1020 		desc, size / 1024);  in run_with_single_pte_of_thp()
1021 	do_run_with_thp(fn, THP_RUN_SINGLE_PTE, size);  in run_with_single_pte_of_thp()
1024 static void run_with_single_pte_of_thp_swap(test_fn fn, const char *desc, size_t size)  in run_with_single_pte_of_thp_swap()  argument
1027 		desc, size / 1024);  in run_with_single_pte_of_thp_swap()
1028 	do_run_with_thp(fn, THP_RUN_SINGLE_PTE_SWAPOUT, size);  in run_with_single_pte_of_thp_swap()
1031 static void run_with_partial_mremap_thp(test_fn fn, const char *desc, size_t size)  in run_with_partial_mremap_thp()  argument
1034 		desc, size / 1024);  in run_with_partial_mremap_thp()
1035 	do_run_with_thp(fn, THP_RUN_PARTIAL_MREMAP, size);  in run_with_partial_mremap_thp()
1038 static void run_with_partial_shared_thp(test_fn fn, const char *desc, size_t size)  in run_with_partial_shared_thp()  argument
1041 		desc, size / 1024);  in run_with_partial_shared_thp()
1042 	do_run_with_thp(fn, THP_RUN_PARTIAL_SHARED, size);  in run_with_partial_shared_thp()
1216 		size_t size = thpsizes[i];  in run_anon_test_case()  local
1220 		settings.hugepages[sz2ord(size)].enabled = THP_ALWAYS;  in run_anon_test_case()
1223 		if (size == pmdsize) {  in run_anon_test_case()
1224 			run_with_thp(test_case->fn, test_case->desc, size);  in run_anon_test_case()
1225 			run_with_thp_swap(test_case->fn, test_case->desc, size);  in run_anon_test_case()
1228 		run_with_pte_mapped_thp(test_case->fn, test_case->desc, size);  in run_anon_test_case()
1229 		run_with_pte_mapped_thp_swap(test_case->fn, test_case->desc, size);  in run_anon_test_case()
1230 		run_with_single_pte_of_thp(test_case->fn, test_case->desc, size);  in run_anon_test_case()
1231 		run_with_single_pte_of_thp_swap(test_case->fn, test_case->desc, size);  in run_anon_test_case()
1232 		run_with_partial_mremap_thp(test_case->fn, test_case->desc, size);  in run_anon_test_case()
1233 		run_with_partial_shared_thp(test_case->fn, test_case->desc, size);  in run_anon_test_case()
1269 static void do_test_anon_thp_collapse(char *mem, size_t size,  in do_test_anon_thp_collapse()  argument
1302 		ret = madvise(mem, size, MADV_COLLAPSE);  in do_test_anon_thp_collapse()
1314 		ret = madvise(mem + size / 2, size / 2, MADV_DONTFORK);  in do_test_anon_thp_collapse()
1323 		ret = madvise(mem, size / 2, MADV_DONTFORK);  in do_test_anon_thp_collapse()
1343 			exit(child_memcmp_fn(mem, size, &comm_pipes));  in do_test_anon_thp_collapse()
1346 			exit(child_memcmp_fn(mem, size / 2, &comm_pipes));  in do_test_anon_thp_collapse()
1349 			exit(child_memcmp_fn(mem + size / 2, size / 2,  in do_test_anon_thp_collapse()
1369 		ret = madvise(mem, size, MADV_DOFORK);  in do_test_anon_thp_collapse()
1380 		ret = madvise(mem, size, MADV_COLLAPSE);  in do_test_anon_thp_collapse()
1394 	memset(mem, 0xff, size);  in do_test_anon_thp_collapse()
1413 static void test_anon_thp_collapse_unshared(char *mem, size_t size,  in test_anon_thp_collapse_unshared()  argument
1417 	do_test_anon_thp_collapse(mem, size, ANON_THP_COLLAPSE_UNSHARED);  in test_anon_thp_collapse_unshared()
1420 static void test_anon_thp_collapse_fully_shared(char *mem, size_t size,  in test_anon_thp_collapse_fully_shared()  argument
1424 	do_test_anon_thp_collapse(mem, size, ANON_THP_COLLAPSE_FULLY_SHARED);  in test_anon_thp_collapse_fully_shared()
1427 static void test_anon_thp_collapse_lower_shared(char *mem, size_t size,  in test_anon_thp_collapse_lower_shared()  argument
1431 	do_test_anon_thp_collapse(mem, size, ANON_THP_COLLAPSE_LOWER_SHARED);  in test_anon_thp_collapse_lower_shared()
1434 static void test_anon_thp_collapse_upper_shared(char *mem, size_t size,  in test_anon_thp_collapse_upper_shared()  argument
1438 	do_test_anon_thp_collapse(mem, size, ANON_THP_COLLAPSE_UPPER_SHARED);  in test_anon_thp_collapse_upper_shared()
1503 typedef void (*non_anon_test_fn)(char *mem, const char *smem, size_t size);
1505 static void test_cow(char *mem, const char *smem, size_t size)  in test_cow()  argument
1507 	char *old = malloc(size);  in test_cow()
1510 	memcpy(old, smem, size);  in test_cow()
1513 	memset(mem, 0xff, size);  in test_cow()
1516 	if (!memcmp(smem, old, size)) {  in test_cow()
1525 static void test_ro_pin(char *mem, const char *smem, size_t size)  in test_ro_pin()  argument
1527 	do_test_ro_pin(mem, size, RO_PIN_TEST, false);  in test_ro_pin()
1530 static void test_ro_fast_pin(char *mem, const char *smem, size_t size)  in test_ro_fast_pin()  argument
1532 	do_test_ro_pin(mem, size, RO_PIN_TEST, true);  in test_ro_fast_pin()