134 static int child_memcmp_fn(char *mem, size_t size,  in child_memcmp_fn()  argument
141 	memcpy(old, mem, size);  in child_memcmp_fn()
149 	return memcmp(old, mem, size);  in child_memcmp_fn()
152 static int child_vmsplice_memcmp_fn(char *mem, size_t size,  in child_vmsplice_memcmp_fn()  argument
156 		.iov_base = mem,  in child_vmsplice_memcmp_fn()
168 	memcpy(old, mem, size);  in child_vmsplice_memcmp_fn()
181 	if (munmap(mem, size) < 0)  in child_vmsplice_memcmp_fn()
199 typedef int (*child_fn)(char *mem, size_t size, struct comm_pipes *comm_pipes);
201 static void do_test_cow_in_parent(char *mem, size_t size, bool do_mprotect,  in do_test_cow_in_parent()  argument
219 		exit(fn(mem, size, &comm_pipes));  in do_test_cow_in_parent()
230 		ret = mprotect(mem, size, PROT_READ);  in do_test_cow_in_parent()
231 		ret |= mprotect(mem, size, PROT_READ|PROT_WRITE);  in do_test_cow_in_parent()
241 	memset(mem, 0xff, size);  in do_test_cow_in_parent()
266 static void test_cow_in_parent(char *mem, size_t size, bool is_hugetlb)  in test_cow_in_parent()  argument
268 	do_test_cow_in_parent(mem, size, false, child_memcmp_fn, false);  in test_cow_in_parent()
271 static void test_cow_in_parent_mprotect(char *mem, size_t size, bool is_hugetlb)  in test_cow_in_parent_mprotect()  argument
273 	do_test_cow_in_parent(mem, size, true, child_memcmp_fn, false);  in test_cow_in_parent_mprotect()
276 static void test_vmsplice_in_child(char *mem, size_t size, bool is_hugetlb)  in test_vmsplice_in_child()  argument
278 	do_test_cow_in_parent(mem, size, false, child_vmsplice_memcmp_fn,  in test_vmsplice_in_child()
282 static void test_vmsplice_in_child_mprotect(char *mem, size_t size,  in test_vmsplice_in_child_mprotect()  argument
285 	do_test_cow_in_parent(mem, size, true, child_vmsplice_memcmp_fn,  in test_vmsplice_in_child_mprotect()
289 static void do_test_vmsplice_in_parent(char *mem, size_t size,  in do_test_vmsplice_in_parent()  argument
293 		.iov_base = mem,  in do_test_vmsplice_in_parent()
305 	memcpy(old, mem, size);  in do_test_vmsplice_in_parent()
335 		memset(mem, 0xff, size);  in do_test_vmsplice_in_parent()
350 	if (munmap(mem, size) < 0) {  in do_test_vmsplice_in_parent()
394 static void test_vmsplice_before_fork(char *mem, size_t size, bool is_hugetlb)  in test_vmsplice_before_fork()  argument
396 	do_test_vmsplice_in_parent(mem, size, true, is_hugetlb);  in test_vmsplice_before_fork()
399 static void test_vmsplice_after_fork(char *mem, size_t size, bool is_hugetlb)  in test_vmsplice_after_fork()  argument
401 	do_test_vmsplice_in_parent(mem, size, false, is_hugetlb);  in test_vmsplice_after_fork()
405 static void do_test_iouring(char *mem, size_t size, bool use_fork)  in do_test_iouring()  argument
451 	iov.iov_base = mem;  in do_test_iouring()
485 		ret = mprotect(mem, size, PROT_READ);  in do_test_iouring()
487 		ret |= mprotect(mem, size, PROT_READ | PROT_WRITE);  in do_test_iouring()
498 	memset(mem, 0xff, size);  in do_test_iouring()
504 	io_uring_prep_write_fixed(sqe, fd, mem, size, 0, 0);  in do_test_iouring()
536 	ksft_test_result(!memcmp(mem, tmp, size),  in do_test_iouring()
556 static void test_iouring_ro(char *mem, size_t size, bool is_hugetlb)  in test_iouring_ro()  argument
558 	do_test_iouring(mem, size, false);  in test_iouring_ro()
561 static void test_iouring_fork(char *mem, size_t size, bool is_hugetlb)  in test_iouring_fork()  argument
563 	do_test_iouring(mem, size, true);  in test_iouring_fork()
575 static void do_test_ro_pin(char *mem, size_t size, enum ro_pin_test test,  in do_test_ro_pin()  argument
645 		ret = mprotect(mem, size, PROT_READ);  in do_test_ro_pin()
647 		ret |= mprotect(mem, size, PROT_READ | PROT_WRITE);  in do_test_ro_pin()
658 	args.addr = (__u64)(uintptr_t)mem;  in do_test_ro_pin()
671 	memset(mem, 0xff, size);  in do_test_ro_pin()
682 		ksft_test_result(!memcmp(mem, tmp, size),  in do_test_ro_pin()
705 static void test_ro_pin_on_shared(char *mem, size_t size, bool is_hugetlb)  in test_ro_pin_on_shared()  argument
707 	do_test_ro_pin(mem, size, RO_PIN_TEST_SHARED, false);  in test_ro_pin_on_shared()
710 static void test_ro_fast_pin_on_shared(char *mem, size_t size, bool is_hugetlb)  in test_ro_fast_pin_on_shared()  argument
712 	do_test_ro_pin(mem, size, RO_PIN_TEST_SHARED, true);  in test_ro_fast_pin_on_shared()
715 static void test_ro_pin_on_ro_previously_shared(char *mem, size_t size,  in test_ro_pin_on_ro_previously_shared()  argument
718 	do_test_ro_pin(mem, size, RO_PIN_TEST_PREVIOUSLY_SHARED, false);  in test_ro_pin_on_ro_previously_shared()
721 static void test_ro_fast_pin_on_ro_previously_shared(char *mem, size_t size,  in test_ro_fast_pin_on_ro_previously_shared()  argument
724 	do_test_ro_pin(mem, size, RO_PIN_TEST_PREVIOUSLY_SHARED, true);  in test_ro_fast_pin_on_ro_previously_shared()
727 static void test_ro_pin_on_ro_exclusive(char *mem, size_t size,  in test_ro_pin_on_ro_exclusive()  argument
730 	do_test_ro_pin(mem, size, RO_PIN_TEST_RO_EXCLUSIVE, false);  in test_ro_pin_on_ro_exclusive()
733 static void test_ro_fast_pin_on_ro_exclusive(char *mem, size_t size,  in test_ro_fast_pin_on_ro_exclusive()  argument
736 	do_test_ro_pin(mem, size, RO_PIN_TEST_RO_EXCLUSIVE, true);  in test_ro_fast_pin_on_ro_exclusive()
739 typedef void (*test_fn)(char *mem, size_t size, bool hugetlb);
743 	char *mem;  in do_run_with_base_page()  local
746 	mem = mmap(NULL, pagesize, PROT_READ | PROT_WRITE,  in do_run_with_base_page()
748 	if (mem == MAP_FAILED) {  in do_run_with_base_page()
753 	ret = madvise(mem, pagesize, MADV_NOHUGEPAGE);  in do_run_with_base_page()
761 	memset(mem, 1, pagesize);  in do_run_with_base_page()
764 		madvise(mem, pagesize, MADV_PAGEOUT);  in do_run_with_base_page()
765 		if (!pagemap_is_swapped(pagemap_fd, mem)) {  in do_run_with_base_page()
771 	fn(mem, pagesize, false);  in do_run_with_base_page()
773 	munmap(mem, pagesize);  in do_run_with_base_page()
801 	char *mem, *mmap_mem, *tmp, *mremap_mem = MAP_FAILED;  in do_run_with_thp()  local
815 	mem = (char *)(((uintptr_t)mmap_mem + thpsize) & ~(thpsize - 1));  in do_run_with_thp()
817 	ret = madvise(mem, thpsize, MADV_HUGEPAGE);  in do_run_with_thp()
827 	mem[0] = 1;  in do_run_with_thp()
828 	if (!pagemap_is_populated(pagemap_fd, mem + thpsize - pagesize)) {  in do_run_with_thp()
832 	memset(mem, 1, thpsize);  in do_run_with_thp()
847 		ret = mprotect(mem + pagesize, pagesize, PROT_READ);  in do_run_with_thp()
852 		ret = mprotect(mem + pagesize, pagesize, PROT_READ | PROT_WRITE);  in do_run_with_thp()
864 		ret = madvise(mem + pagesize, thpsize - pagesize, MADV_DONTNEED);  in do_run_with_thp()
879 		if (mem == MAP_FAILED) {  in do_run_with_thp()
883 		tmp = mremap(mem + mremap_size, mremap_size, mremap_size,  in do_run_with_thp()
897 		ret = madvise(mem + pagesize, thpsize - pagesize, MADV_DONTFORK);  in do_run_with_thp()
911 		ret = madvise(mem + pagesize, thpsize - pagesize, MADV_DOFORK);  in do_run_with_thp()
925 		madvise(mem, size, MADV_PAGEOUT);  in do_run_with_thp()
926 		if (!range_is_swapped(mem, size)) {  in do_run_with_thp()
935 	fn(mem, size, false);  in do_run_with_thp()
1001 	char *mem, *dummy;  in run_with_hugetlb()  local
1008 	mem = mmap(NULL, hugetlbsize, PROT_READ | PROT_WRITE, flags, -1, 0);  in run_with_hugetlb()
1009 	if (mem == MAP_FAILED) {  in run_with_hugetlb()
1015 	memset(mem, 1, hugetlbsize);  in run_with_hugetlb()
1028 	fn(mem, hugetlbsize, true);  in run_with_hugetlb()
1030 	munmap(mem, hugetlbsize);  in run_with_hugetlb()
1220 static void do_test_anon_thp_collapse(char *mem, size_t size,  in do_test_anon_thp_collapse()  argument
1237 	ret = mprotect(mem + pagesize, pagesize, PROT_READ);  in do_test_anon_thp_collapse()
1242 	ret = mprotect(mem + pagesize, pagesize, PROT_READ | PROT_WRITE);  in do_test_anon_thp_collapse()
1251 		ret = madvise(mem, size, MADV_COLLAPSE);  in do_test_anon_thp_collapse()
1263 		ret = madvise(mem + size / 2, size / 2, MADV_DONTFORK);  in do_test_anon_thp_collapse()
1271 		ret = madvise(mem, size / 2, MADV_DONTFORK);  in do_test_anon_thp_collapse()
1289 			exit(child_memcmp_fn(mem, size, &comm_pipes));  in do_test_anon_thp_collapse()
1292 			exit(child_memcmp_fn(mem, size / 2, &comm_pipes));  in do_test_anon_thp_collapse()
1295 			exit(child_memcmp_fn(mem + size / 2, size / 2,  in do_test_anon_thp_collapse()
1315 		ret = madvise(mem, size, MADV_DOFORK);  in do_test_anon_thp_collapse()
1325 		ret = madvise(mem, size, MADV_COLLAPSE);  in do_test_anon_thp_collapse()
1339 	memset(mem, 0xff, size);  in do_test_anon_thp_collapse()
1353 static void test_anon_thp_collapse_unshared(char *mem, size_t size,  in test_anon_thp_collapse_unshared()  argument
1357 	do_test_anon_thp_collapse(mem, size, ANON_THP_COLLAPSE_UNSHARED);  in test_anon_thp_collapse_unshared()
1360 static void test_anon_thp_collapse_fully_shared(char *mem, size_t size,  in test_anon_thp_collapse_fully_shared()  argument
1364 	do_test_anon_thp_collapse(mem, size, ANON_THP_COLLAPSE_FULLY_SHARED);  in test_anon_thp_collapse_fully_shared()
1367 static void test_anon_thp_collapse_lower_shared(char *mem, size_t size,  in test_anon_thp_collapse_lower_shared()  argument
1371 	do_test_anon_thp_collapse(mem, size, ANON_THP_COLLAPSE_LOWER_SHARED);  in test_anon_thp_collapse_lower_shared()
1374 static void test_anon_thp_collapse_upper_shared(char *mem, size_t size,  in test_anon_thp_collapse_upper_shared()  argument
1378 	do_test_anon_thp_collapse(mem, size, ANON_THP_COLLAPSE_UPPER_SHARED);  in test_anon_thp_collapse_upper_shared()
1443 typedef void (*non_anon_test_fn)(char *mem, const char *smem, size_t size);
1445 static void test_cow(char *mem, const char *smem, size_t size)  in test_cow()  argument
1453 	memset(mem, 0xff, size);  in test_cow()
1461 static void test_ro_pin(char *mem, const char *smem, size_t size)  in test_ro_pin()  argument
1463 	do_test_ro_pin(mem, size, RO_PIN_TEST, false);  in test_ro_pin()
1466 static void test_ro_fast_pin(char *mem, const char *smem, size_t size)  in test_ro_fast_pin()  argument
1468 	do_test_ro_pin(mem, size, RO_PIN_TEST, true);  in test_ro_fast_pin()
1473 	char *mem, *smem, tmp;  in run_with_zeropage()  local
1477 	mem = mmap(NULL, pagesize, PROT_READ | PROT_WRITE,  in run_with_zeropage()
1479 	if (mem == MAP_FAILED) {  in run_with_zeropage()
1491 	tmp = *mem + *smem;  in run_with_zeropage()
1494 	fn(mem, smem, pagesize);  in run_with_zeropage()
1496 	munmap(mem, pagesize);  in run_with_zeropage()
1503 	char *mem, *smem, *mmap_mem, *mmap_smem, tmp;  in run_with_huge_zeropage()  local
1530 	mem = (char *)(((uintptr_t)mmap_mem + pmdsize) & ~(pmdsize - 1));  in run_with_huge_zeropage()
1533 	ret = madvise(mem, pmdsize, MADV_HUGEPAGE);  in run_with_huge_zeropage()
1545 	tmp = *mem + *smem;  in run_with_huge_zeropage()
1547 	if (!pagemap_is_populated(pagemap_fd, mem + pagesize) ||  in run_with_huge_zeropage()
1553 	fn(mem, smem, pmdsize);  in run_with_huge_zeropage()
1562 	char *mem, *smem, tmp;  in run_with_memfd()  local
1580 	mem = mmap(NULL, pagesize, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);  in run_with_memfd()
1581 	if (mem == MAP_FAILED) {  in run_with_memfd()
1592 	tmp = *mem + *smem;  in run_with_memfd()
1595 	fn(mem, smem, pagesize);  in run_with_memfd()
1597 	munmap(mem, pagesize);  in run_with_memfd()
1606 	char *mem, *smem, tmp;  in run_with_tmpfile()  local
1631 	mem = mmap(NULL, pagesize, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);  in run_with_tmpfile()
1632 	if (mem == MAP_FAILED) {  in run_with_tmpfile()
1643 	tmp = *mem + *smem;  in run_with_tmpfile()
1646 	fn(mem, smem, pagesize);  in run_with_tmpfile()
1648 	munmap(mem, pagesize);  in run_with_tmpfile()
1659 	char *mem, *smem, tmp;  in run_with_memfd_hugetlb()  local
1680 	mem = mmap(NULL, hugetlbsize, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd,  in run_with_memfd_hugetlb()
1682 	if (mem == MAP_FAILED) {  in run_with_memfd_hugetlb()
1693 	tmp = *mem + *smem;  in run_with_memfd_hugetlb()
1696 	fn(mem, smem, hugetlbsize);  in run_with_memfd_hugetlb()
1698 	munmap(mem, hugetlbsize);  in run_with_memfd_hugetlb()