Lines Matching +full:page +full:- +full:size
1 // SPDX-License-Identifier: GPL-2.0-only
44 static int sz2ord(size_t size)
46 return __builtin_ctzll(size / pagesize);
69 ksft_print_msg("[INFO] detected THP size: %zu KiB\n", kb);
100 static bool range_is_swapped(void *addr, size_t size)
102 for (; size; addr += pagesize, size -= pagesize)
115 if (pipe(comm_pipes->child_ready) < 0)
116 return -errno;
117 if (pipe(comm_pipes->parent_ready) < 0) {
118 close(comm_pipes->child_ready[0]);
119 close(comm_pipes->child_ready[1]);
120 return -errno;
128 close(comm_pipes->child_ready[0]);
129 close(comm_pipes->child_ready[1]);
130 close(comm_pipes->parent_ready[0]);
131 close(comm_pipes->parent_ready[1]);
134 static int child_memcmp_fn(char *mem, size_t size,
137 char *old = malloc(size);
141 memcpy(old, mem, size);
143 /* Wait until the parent modified the page. */
144 write(comm_pipes->child_ready[1], "0", 1);
145 while (read(comm_pipes->parent_ready[0], &buf, 1) != 1)
149 return memcmp(old, mem, size);
152 static int child_vmsplice_memcmp_fn(char *mem, size_t size,
157 .iov_len = size,
164 old = malloc(size);
165 new = malloc(size);
168 memcpy(old, mem, size);
171 return -errno;
173 /* Trigger a read-only pin. */
176 return -errno;
178 return -EINVAL;
180 /* Unmap it from our page tables. */
181 if (munmap(mem, size) < 0)
182 return -errno;
185 write(comm_pipes->child_ready[1], "0", 1);
186 while (read(comm_pipes->parent_ready[0], &buf, 1) != 1)
191 cur = read(fds[0], new + total, transferred - total);
193 return -errno;
199 typedef int (*child_fn)(char *mem, size_t size, struct comm_pipes *comm_pipes);
201 static void do_test_cow_in_parent(char *mem, size_t size, bool do_mprotect,
219 exit(fn(mem, size, &comm_pipes));
228 * write-faults by directly mapping pages writable.
230 ret = mprotect(mem, size, PROT_READ);
231 ret |= mprotect(mem, size, PROT_READ|PROT_WRITE);
240 /* Modify the page. */
241 memset(mem, 0xff, size);
248 ret = -EINVAL;
266 static void test_cow_in_parent(char *mem, size_t size, bool is_hugetlb)
268 do_test_cow_in_parent(mem, size, false, child_memcmp_fn, false);
271 static void test_cow_in_parent_mprotect(char *mem, size_t size, bool is_hugetlb)
273 do_test_cow_in_parent(mem, size, true, child_memcmp_fn, false);
276 static void test_vmsplice_in_child(char *mem, size_t size, bool is_hugetlb)
278 do_test_cow_in_parent(mem, size, false, child_vmsplice_memcmp_fn,
282 static void test_vmsplice_in_child_mprotect(char *mem, size_t size,
285 do_test_cow_in_parent(mem, size, true, child_vmsplice_memcmp_fn,
289 static void do_test_vmsplice_in_parent(char *mem, size_t size,
294 .iov_len = size,
302 old = malloc(size);
303 new = malloc(size);
305 memcpy(old, mem, size);
334 /* Modify page content in the child. */
335 memset(mem, 0xff, size);
350 if (munmap(mem, size) < 0) {
365 cur = read(fds[0], new + total, transferred - total);
394 static void test_vmsplice_before_fork(char *mem, size_t size, bool is_hugetlb)
396 do_test_vmsplice_in_parent(mem, size, true, is_hugetlb);
399 static void test_vmsplice_after_fork(char *mem, size_t size, bool is_hugetlb)
401 do_test_vmsplice_in_parent(mem, size, false, is_hugetlb);
405 static void do_test_iouring(char *mem, size_t size, bool use_fork)
431 tmp = malloc(size);
452 iov.iov_len = size;
462 * we expect the pinned page to not get shared with the child.
479 * Map the page R/O into the page table. Enable softdirty
480 * tracking to stop the page from getting mapped R/W immediately
483 * if the page is mapped R/O vs. R/W).
485 ret = mprotect(mem, size, PROT_READ);
487 ret |= mprotect(mem, size, PROT_READ | PROT_WRITE);
495 * Modify the page and write page content as observed by the fixed
498 memset(mem, 0xff, size);
504 io_uring_prep_write_fixed(sqe, fd, mem, size, 0, 0);
518 if (cqe->res != size) {
526 while (total < size) {
527 cur = pread(fd, tmp + total, size - total, total);
536 ksft_test_result(!memcmp(mem, tmp, size),
556 static void test_iouring_ro(char *mem, size_t size, bool is_hugetlb)
558 do_test_iouring(mem, size, false);
561 static void test_iouring_fork(char *mem, size_t size, bool is_hugetlb)
563 do_test_iouring(mem, size, true);
575 static void do_test_ro_pin(char *mem, size_t size, enum ro_pin_test test,
589 tmp = malloc(size);
628 * The pages should now be mapped R/O into our page
639 * Map the page R/O into the page table. Enable softdirty
640 * tracking to stop the page from getting mapped R/W immediately
643 * if the page is mapped R/O vs. R/W).
645 ret = mprotect(mem, size, PROT_READ);
647 ret |= mprotect(mem, size, PROT_READ | PROT_WRITE);
659 args.size = size;
670 /* Modify the page. */
671 memset(mem, 0xff, size);
682 ksft_test_result(!memcmp(mem, tmp, size),
705 static void test_ro_pin_on_shared(char *mem, size_t size, bool is_hugetlb)
707 do_test_ro_pin(mem, size, RO_PIN_TEST_SHARED, false);
710 static void test_ro_fast_pin_on_shared(char *mem, size_t size, bool is_hugetlb)
712 do_test_ro_pin(mem, size, RO_PIN_TEST_SHARED, true);
715 static void test_ro_pin_on_ro_previously_shared(char *mem, size_t size,
718 do_test_ro_pin(mem, size, RO_PIN_TEST_PREVIOUSLY_SHARED, false);
721 static void test_ro_fast_pin_on_ro_previously_shared(char *mem, size_t size,
724 do_test_ro_pin(mem, size, RO_PIN_TEST_PREVIOUSLY_SHARED, true);
727 static void test_ro_pin_on_ro_exclusive(char *mem, size_t size,
730 do_test_ro_pin(mem, size, RO_PIN_TEST_RO_EXCLUSIVE, false);
733 static void test_ro_fast_pin_on_ro_exclusive(char *mem, size_t size,
736 do_test_ro_pin(mem, size, RO_PIN_TEST_RO_EXCLUSIVE, true);
739 typedef void (*test_fn)(char *mem, size_t size, bool hugetlb);
747 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
760 /* Populate a base page. */
778 ksft_print_msg("[RUN] %s ... with base page\n", desc);
784 ksft_print_msg("[RUN] %s ... with swapped out base page\n", desc);
802 size_t size, mmap_size, mremap_size;
805 /* For alignment purposes, we need twice the thp size. */
808 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
814 /* We need a THP-aligned memory area. */
815 mem = (char *)(((uintptr_t)mmap_mem + thpsize) & ~(thpsize - 1));
824 * Try to populate a THP. Touch the first sub-page and test if
825 * we get the last sub-page populated automatically.
828 if (!pagemap_is_populated(pagemap_fd, mem + thpsize - pagesize)) {
834 size = thpsize;
843 * Trigger PTE-mapping the THP by temporarily mapping a single
845 * therefore already PTE-mapped).
861 * Discard all but a single subpage of that PTE-mapped THP. What
864 ret = madvise(mem + pagesize, thpsize - pagesize, MADV_DONTNEED);
869 size = pagesize;
878 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
889 size = mremap_size;
893 * Share the first page of the THP with a child and quit the
897 ret = madvise(mem + pagesize, thpsize - pagesize, MADV_DONTFORK);
911 ret = madvise(mem + pagesize, thpsize - pagesize, MADV_DOFORK);
925 madvise(mem, size, MADV_PAGEOUT);
926 if (!range_is_swapped(mem, size)) {
935 fn(mem, size, false);
942 static void run_with_thp(test_fn fn, const char *desc, size_t size)
945 desc, size / 1024);
946 do_run_with_thp(fn, THP_RUN_PMD, size);
949 static void run_with_thp_swap(test_fn fn, const char *desc, size_t size)
951 ksft_print_msg("[RUN] %s ... with swapped-out THP (%zu kB)\n",
952 desc, size / 1024);
953 do_run_with_thp(fn, THP_RUN_PMD_SWAPOUT, size);
956 static void run_with_pte_mapped_thp(test_fn fn, const char *desc, size_t size)
958 ksft_print_msg("[RUN] %s ... with PTE-mapped THP (%zu kB)\n",
959 desc, size / 1024);
960 do_run_with_thp(fn, THP_RUN_PTE, size);
963 static void run_with_pte_mapped_thp_swap(test_fn fn, const char *desc, size_t size)
965 ksft_print_msg("[RUN] %s ... with swapped-out, PTE-mapped THP (%zu kB)\n",
966 desc, size / 1024);
967 do_run_with_thp(fn, THP_RUN_PTE_SWAPOUT, size);
970 static void run_with_single_pte_of_thp(test_fn fn, const char *desc, size_t size)
973 desc, size / 1024);
974 do_run_with_thp(fn, THP_RUN_SINGLE_PTE, size);
977 static void run_with_single_pte_of_thp_swap(test_fn fn, const char *desc, size_t size)
979 ksft_print_msg("[RUN] %s ... with single PTE of swapped-out THP (%zu kB)\n",
980 desc, size / 1024);
981 do_run_with_thp(fn, THP_RUN_SINGLE_PTE_SWAPOUT, size);
984 static void run_with_partial_mremap_thp(test_fn fn, const char *desc, size_t size)
987 desc, size / 1024);
988 do_run_with_thp(fn, THP_RUN_PARTIAL_MREMAP, size);
991 static void run_with_partial_shared_thp(test_fn fn, const char *desc, size_t size)
994 desc, size / 1024);
995 do_run_with_thp(fn, THP_RUN_PARTIAL_SHARED, size);
1008 mem = mmap(NULL, hugetlbsize, PROT_READ | PROT_WRITE, flags, -1, 0);
1014 /* Populate an huge page. */
1021 dummy = mmap(NULL, hugetlbsize, PROT_READ | PROT_WRITE, flags, -1, 0);
1063 * This is CVE-2020-29374 reported by Jann Horn.
1097 * Take a R/W longterm pin and then map the page R/O into the page
1099 * page, the page content must be visible via the pin.
1102 "R/O-mapping a page registered as iouring fixed buffer",
1107 * page, the page content must be visible via the pin. We expect the
1108 * pinned page to not get shared with the child.
1117 * Take a R/O longterm pin on a R/O-mapped shared anonymous page.
1118 * When modifying the page via the page table, the page content change
1122 "R/O GUP pin on R/O-mapped shared page",
1125 /* Same as above, but using GUP-fast. */
1127 "R/O GUP-fast pin on R/O-mapped shared page",
1131 * Take a R/O longterm pin on a R/O-mapped exclusive anonymous page that
1132 * was previously shared. When modifying the page via the page table,
1133 * the page content change must be visible via the pin.
1136 "R/O GUP pin on R/O-mapped previously-shared page",
1139 /* Same as above, but using GUP-fast. */
1141 "R/O GUP-fast pin on R/O-mapped previously-shared page",
1145 * Take a R/O longterm pin on a R/O-mapped exclusive anonymous page.
1146 * When modifying the page via the page table, the page content change
1150 "R/O GUP pin on R/O-mapped exclusive page",
1153 /* Same as above, but using GUP-fast. */
1155 "R/O GUP-fast pin on R/O-mapped exclusive page",
1164 run_with_base_page(test_case->fn, test_case->desc);
1165 run_with_base_page_swap(test_case->fn, test_case->desc);
1167 size_t size = thpsizes[i];
1171 settings.hugepages[sz2ord(size)].enabled = THP_ALWAYS;
1174 if (size == pmdsize) {
1175 run_with_thp(test_case->fn, test_case->desc, size);
1176 run_with_thp_swap(test_case->fn, test_case->desc, size);
1179 run_with_pte_mapped_thp(test_case->fn, test_case->desc, size);
1180 run_with_pte_mapped_thp_swap(test_case->fn, test_case->desc, size);
1181 run_with_single_pte_of_thp(test_case->fn, test_case->desc, size);
1182 run_with_single_pte_of_thp_swap(test_case->fn, test_case->desc, size);
1183 run_with_partial_mremap_thp(test_case->fn, test_case->desc, size);
1184 run_with_partial_shared_thp(test_case->fn, test_case->desc, size);
1189 run_with_hugetlb(test_case->fn, test_case->desc,
1220 static void do_test_anon_thp_collapse(char *mem, size_t size,
1234 * Trigger PTE-mapping the THP by temporarily mapping a single subpage
1250 /* Collapse before actually COW-sharing the page. */
1251 ret = madvise(mem, size, MADV_COLLAPSE);
1259 /* COW-share the full PTE-mapped THP. */
1262 /* Don't COW-share the upper part of the THP. */
1263 ret = madvise(mem + size / 2, size / 2, MADV_DONTFORK);
1270 /* Don't COW-share the lower part of the THP. */
1271 ret = madvise(mem, size / 2, MADV_DONTFORK);
1289 exit(child_memcmp_fn(mem, size, &comm_pipes));
1292 exit(child_memcmp_fn(mem, size / 2, &comm_pipes));
1295 exit(child_memcmp_fn(mem + size / 2, size / 2,
1315 ret = madvise(mem, size, MADV_DOFORK);
1324 /* Collapse before anyone modified the COW-shared page. */
1325 ret = madvise(mem, size, MADV_COLLAPSE);
1338 /* Modify the page. */
1339 memset(mem, 0xff, size);
1346 ret = -EINVAL;
1353 static void test_anon_thp_collapse_unshared(char *mem, size_t size,
1357 do_test_anon_thp_collapse(mem, size, ANON_THP_COLLAPSE_UNSHARED);
1360 static void test_anon_thp_collapse_fully_shared(char *mem, size_t size,
1364 do_test_anon_thp_collapse(mem, size, ANON_THP_COLLAPSE_FULLY_SHARED);
1367 static void test_anon_thp_collapse_lower_shared(char *mem, size_t size,
1371 do_test_anon_thp_collapse(mem, size, ANON_THP_COLLAPSE_LOWER_SHARED);
1374 static void test_anon_thp_collapse_upper_shared(char *mem, size_t size,
1378 do_test_anon_thp_collapse(mem, size, ANON_THP_COLLAPSE_UPPER_SHARED);
1390 * Re-mapping a PTE-mapped anon THP using a single PMD ("in-place
1398 /* Basic COW test, but collapse after COW-sharing a full THP. */
1404 * Basic COW test, but collapse after COW-sharing the lower half of a
1412 * Basic COW test, but collapse after COW-sharing the upper half of a
1433 ksft_print_msg("[RUN] %s\n", test_case->desc);
1434 do_run_with_thp(test_case->fn, THP_RUN_PMD, pmdsize);
1443 typedef void (*non_anon_test_fn)(char *mem, const char *smem, size_t size);
1445 static void test_cow(char *mem, const char *smem, size_t size)
1447 char *old = malloc(size);
1450 memcpy(old, smem, size);
1452 /* Modify the page. */
1453 memset(mem, 0xff, size);
1456 ksft_test_result(!memcmp(smem, old, size),
1461 static void test_ro_pin(char *mem, const char *smem, size_t size)
1463 do_test_ro_pin(mem, size, RO_PIN_TEST, false);
1466 static void test_ro_fast_pin(char *mem, const char *smem, size_t size)
1468 do_test_ro_pin(mem, size, RO_PIN_TEST, true);
1478 MAP_PRIVATE | MAP_ANON, -1, 0);
1484 smem = mmap(NULL, pagesize, PROT_READ, MAP_PRIVATE | MAP_ANON, -1, 0);
1490 /* Read from the page to populate the shared zeropage. */
1514 /* For alignment purposes, we need twice the thp size. */
1517 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
1523 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
1529 /* We need a THP-aligned memory area. */
1530 mem = (char *)(((uintptr_t)mmap_mem + pmdsize) & ~(pmdsize - 1));
1531 smem = (char *)(((uintptr_t)mmap_smem + pmdsize) & ~(pmdsize - 1));
1542 * the first sub-page and test if we get another sub-page populated
1573 /* File consists of a single page filled with zeroes. */
1591 /* Fault the page in. */
1624 /* File consists of a single page filled with zeroes. */
1642 /* Fault the page in. */
1673 /* File consists of a single page filled with zeroes. */
1692 /* Fault the page in. */
1725 * Take a R/O longterm pin. When modifying the page via the page table,
1726 * the page content change must be visible via the pin.
1732 /* Same as above, but using GUP-fast. */
1734 "R/O longterm GUP-fast pin",
1743 run_with_zeropage(test_case->fn, test_case->desc);
1744 run_with_memfd(test_case->fn, test_case->desc);
1745 run_with_tmpfile(test_case->fn, test_case->desc);
1747 run_with_huge_zeropage(test_case->fn, test_case->desc);
1749 run_with_memfd_hugetlb(test_case->fn, test_case->desc,
1757 ksft_print_msg("[RUN] Non-anonymous memory tests in private mappings\n");
1788 ksft_print_msg("[INFO] detected PMD size: %zu KiB\n",