1 // SPDX-License-Identifier: GPL-2.0-only
102 	for (; size; addr += pagesize, size -= pagesize)  in range_is_swapped()
115 	if (pipe(comm_pipes->child_ready) < 0)  in setup_comm_pipes()
116 		return -errno;  in setup_comm_pipes()
117 	if (pipe(comm_pipes->parent_ready) < 0) {  in setup_comm_pipes()
118 		close(comm_pipes->child_ready[0]);  in setup_comm_pipes()
119 		close(comm_pipes->child_ready[1]);  in setup_comm_pipes()
120 		return -errno;  in setup_comm_pipes()
128 	close(comm_pipes->child_ready[0]);  in close_comm_pipes()
129 	close(comm_pipes->child_ready[1]);  in close_comm_pipes()
130 	close(comm_pipes->parent_ready[0]);  in close_comm_pipes()
131 	close(comm_pipes->parent_ready[1]);  in close_comm_pipes()
144 	write(comm_pipes->child_ready[1], "0", 1);  in child_memcmp_fn()
145 	while (read(comm_pipes->parent_ready[0], &buf, 1) != 1)  in child_memcmp_fn()
171 		return -errno;  in child_vmsplice_memcmp_fn()
173 	/* Trigger a read-only pin. */  in child_vmsplice_memcmp_fn()
176 		return -errno;  in child_vmsplice_memcmp_fn()
178 		return -EINVAL;  in child_vmsplice_memcmp_fn()
182 		return -errno;  in child_vmsplice_memcmp_fn()
185 	write(comm_pipes->child_ready[1], "0", 1);  in child_vmsplice_memcmp_fn()
186 	while (read(comm_pipes->parent_ready[0], &buf, 1) != 1)  in child_vmsplice_memcmp_fn()
191 		cur = read(fds[0], new + total, transferred - total);  in child_vmsplice_memcmp_fn()
193 			return -errno;  in child_vmsplice_memcmp_fn()
228 		 * write-faults by directly mapping pages writable.  in do_test_cow_in_parent()
248 		ret = -EINVAL;  in do_test_cow_in_parent()
365 		cur = read(fds[0], new + total, transferred - total);  in do_test_vmsplice_in_parent()
462 		 * we expect the pinned page to not get shared with the child.  in do_test_iouring()
496 	 * buffer pin to the file so we can verify it.  in do_test_iouring()
518 	if (cqe->res != size) {  in do_test_iouring()
527 		cur = pread(fd, tmp + total, size - total, total);  in do_test_iouring()
537 			 "Longterm R/W pin is reliable\n");  in do_test_iouring()
629 			 * tables, but they are no longer shared.  in do_test_ro_pin()
657 	/* Take a R/O pin. This should trigger unsharing. */  in do_test_ro_pin()
674 	 * Read back the content via the pin to the temporary buffer and  in do_test_ro_pin()
683 				 "Longterm R/O pin is reliable\n");  in do_test_ro_pin()
747 		   MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);  in do_run_with_base_page()
808 			MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);  in do_run_with_thp()
814 	/* We need a THP-aligned memory area. */  in do_run_with_thp()
815 	mem = (char *)(((uintptr_t)mmap_mem + thpsize) & ~(thpsize - 1));  in do_run_with_thp()
824 	 * Try to populate a THP. Touch the first sub-page and test if  in do_run_with_thp()
825 	 * we get the last sub-page populated automatically.  in do_run_with_thp()
828 	if (!pagemap_is_populated(pagemap_fd, mem + thpsize - pagesize)) {  in do_run_with_thp()
843 		 * Trigger PTE-mapping the THP by temporarily mapping a single  in do_run_with_thp()
845 		 * therefore already PTE-mapped).  in do_run_with_thp()
861 		 * Discard all but a single subpage of that PTE-mapped THP. What  in do_run_with_thp()
864 		ret = madvise(mem + pagesize, thpsize - pagesize, MADV_DONTNEED);  in do_run_with_thp()
878 				  MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);  in do_run_with_thp()
895 		 * have been shared.  in do_run_with_thp()
897 		ret = madvise(mem + pagesize, thpsize - pagesize, MADV_DONTFORK);  in do_run_with_thp()
911 		ret = madvise(mem + pagesize, thpsize - pagesize, MADV_DOFORK);  in do_run_with_thp()
951 	ksft_print_msg("[RUN] %s ... with swapped-out THP (%zu kB)\n",  in run_with_thp_swap()
958 	ksft_print_msg("[RUN] %s ... with PTE-mapped THP (%zu kB)\n",  in run_with_pte_mapped_thp()
965 	ksft_print_msg("[RUN] %s ... with swapped-out, PTE-mapped THP (%zu kB)\n",  in run_with_pte_mapped_thp_swap()
979 	ksft_print_msg("[RUN] %s ... with single PTE of swapped-out THP (%zu kB)\n",  in run_with_single_pte_of_thp_swap()
993 	ksft_print_msg("[RUN] %s ... with partially shared THP (%zu kB)\n",  in run_with_partial_shared_thp()
1008 	mem = mmap(NULL, hugetlbsize, PROT_READ | PROT_WRITE, flags, -1, 0);  in run_with_hugetlb()
1021 	dummy = mmap(NULL, hugetlbsize, PROT_READ | PROT_WRITE, flags, -1, 0);  in run_with_hugetlb()
1040  * that may get shared via COW during fork().
1063 	 * This is CVE-2020-29374 reported by Jann Horn.
1097 	 * Take a R/W longterm pin and then map the page R/O into the page
1099 	 * page, the page content must be visible via the pin.
1102 		"R/O-mapping a page registered as iouring fixed buffer",
1106 	 * Take a R/W longterm pin and then fork() a child. When modifying the
1107 	 * page, the page content must be visible via the pin. We expect the
1108 	 * pinned page to not get shared with the child.
1117 	 * Take a R/O longterm pin on a R/O-mapped shared anonymous page.
1119 	 * must be visible via the pin.
1122 		"R/O GUP pin on R/O-mapped shared page",
1125 	/* Same as above, but using GUP-fast. */
1127 		"R/O GUP-fast pin on R/O-mapped shared page",
1131 	 * Take a R/O longterm pin on a R/O-mapped exclusive anonymous page that
1132 	 * was previously shared. When modifying the page via the page table,
1133 	 * the page content change must be visible via the pin.
1136 		"R/O GUP pin on R/O-mapped previously-shared page",
1139 	/* Same as above, but using GUP-fast. */
1141 		"R/O GUP-fast pin on R/O-mapped previously-shared page",
1145 	 * Take a R/O longterm pin on a R/O-mapped exclusive anonymous page.
1147 	 * must be visible via the pin.
1150 		"R/O GUP pin on R/O-mapped exclusive page",
1153 	/* Same as above, but using GUP-fast. */
1155 		"R/O GUP-fast pin on R/O-mapped exclusive page",
1164 	run_with_base_page(test_case->fn, test_case->desc);  in run_anon_test_case()
1165 	run_with_base_page_swap(test_case->fn, test_case->desc);  in run_anon_test_case()
1175 			run_with_thp(test_case->fn, test_case->desc, size);  in run_anon_test_case()
1176 			run_with_thp_swap(test_case->fn, test_case->desc, size);  in run_anon_test_case()
1179 		run_with_pte_mapped_thp(test_case->fn, test_case->desc, size);  in run_anon_test_case()
1180 		run_with_pte_mapped_thp_swap(test_case->fn, test_case->desc, size);  in run_anon_test_case()
1181 		run_with_single_pte_of_thp(test_case->fn, test_case->desc, size);  in run_anon_test_case()
1182 		run_with_single_pte_of_thp_swap(test_case->fn, test_case->desc, size);  in run_anon_test_case()
1183 		run_with_partial_mremap_thp(test_case->fn, test_case->desc, size);  in run_anon_test_case()
1184 		run_with_partial_shared_thp(test_case->fn, test_case->desc, size);  in run_anon_test_case()
1189 		run_with_hugetlb(test_case->fn, test_case->desc,  in run_anon_test_case()
1234 	 * Trigger PTE-mapping the THP by temporarily mapping a single subpage  in do_test_anon_thp_collapse()
1250 		/* Collapse before actually COW-sharing the page. */  in do_test_anon_thp_collapse()
1259 		/* COW-share the full PTE-mapped THP. */  in do_test_anon_thp_collapse()
1262 		/* Don't COW-share the upper part of the THP. */  in do_test_anon_thp_collapse()
1270 		/* Don't COW-share the lower part of the THP. */  in do_test_anon_thp_collapse()
1324 		/* Collapse before anyone modified the COW-shared page. */  in do_test_anon_thp_collapse()
1346 		ret = -EINVAL;  in do_test_anon_thp_collapse()
1383  * that may get shared via COW during fork().
1390 	 * Re-mapping a PTE-mapped anon THP using a single PMD ("in-place
1398 	/* Basic COW test, but collapse after COW-sharing a full THP. */
1400 		"Basic COW after fork() when collapsing after fork() (fully shared)",
1404 	 * Basic COW test, but collapse after COW-sharing the lower half of a
1408 		"Basic COW after fork() when collapsing after fork() (lower shared)",
1412 	 * Basic COW test, but collapse after COW-sharing the upper half of a
1416 		"Basic COW after fork() when collapsing after fork() (upper shared)",
1433 		ksft_print_msg("[RUN] %s\n", test_case->desc);  in run_anon_thp_test_cases()
1434 		do_run_with_thp(test_case->fn, THP_RUN_PMD, pmdsize);  in run_anon_thp_test_cases()
1475 	ksft_print_msg("[RUN] %s ... with shared zeropage\n", desc);  in run_with_zeropage()
1478 		   MAP_PRIVATE | MAP_ANON, -1, 0);  in run_with_zeropage()
1484 	smem = mmap(NULL, pagesize, PROT_READ, MAP_PRIVATE | MAP_ANON, -1, 0);  in run_with_zeropage()
1490 	/* Read from the page to populate the shared zeropage. */  in run_with_zeropage()
1517 			MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);  in run_with_huge_zeropage()
1523 			 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);  in run_with_huge_zeropage()
1529 	/* We need a THP-aligned memory area. */  in run_with_huge_zeropage()
1530 	mem = (char *)(((uintptr_t)mmap_mem + pmdsize) & ~(pmdsize - 1));  in run_with_huge_zeropage()
1531 	smem = (char *)(((uintptr_t)mmap_smem + pmdsize) & ~(pmdsize - 1));  in run_with_huge_zeropage()
1541 	 * Read from the memory to populate the huge shared zeropage. Read from  in run_with_huge_zeropage()
1542 	 * the first sub-page and test if we get another sub-page populated  in run_with_huge_zeropage()
1712  * pages that may get shared via COW ndependent of fork(). This includes
1713  * the shared zeropage(s), pagecache pages, ...
1718 	 * visible via other private/shared mappings.
1725 	 * Take a R/O longterm pin. When modifying the page via the page table,
1726 	 * the page content change must be visible via the pin.
1729 		"R/O longterm GUP pin",
1732 	/* Same as above, but using GUP-fast. */
1734 		"R/O longterm GUP-fast pin",
1743 	run_with_zeropage(test_case->fn, test_case->desc);  in run_non_anon_test_case()
1744 	run_with_memfd(test_case->fn, test_case->desc);  in run_non_anon_test_case()
1745 	run_with_tmpfile(test_case->fn, test_case->desc);  in run_non_anon_test_case()
1747 		run_with_huge_zeropage(test_case->fn, test_case->desc);  in run_non_anon_test_case()
1749 		run_with_memfd_hugetlb(test_case->fn, test_case->desc,  in run_non_anon_test_case()
1757 	ksft_print_msg("[RUN] Non-anonymous memory tests in private mappings\n");  in run_non_anon_test_cases()