1 // SPDX-License-Identifier: GPL-2.0-only 2 3 #define _GNU_SOURCE 4 #include <stdbool.h> 5 #include <stdint.h> 6 #include <fcntl.h> 7 #include <assert.h> 8 #include <linux/mman.h> 9 #include <sys/mman.h> 10 #include "kselftest.h" 11 #include "hugepage_settings.h" 12 #include "uffd-common.h" 13 14 static int pagemap_fd; 15 static int nr_pagesizes = 1; 16 static unsigned long pagesize; 17 static int nr_thpsizes; 18 static size_t thpsizes[20]; 19 static int nr_hugetlbsizes; 20 static unsigned long hugetlbsizes[10]; 21 22 static void check_uffd_wp_feature_supported(void) 23 { 24 uint64_t features = 0; 25 26 if (uffd_get_features(&features)) 27 ksft_exit_skip("failed to get available features (%d)\n", errno); 28 29 if (!(features & UFFD_FEATURE_PAGEFAULT_FLAG_WP)) 30 ksft_exit_skip("uffd-wp feature not supported\n"); 31 } 32 33 static int detect_thp_sizes(size_t sizes[], int max) 34 { 35 int count = 0; 36 unsigned long orders; 37 size_t kb; 38 int i; 39 40 /* thp not supported at all. */ 41 if (!read_pmd_pagesize()) 42 return 0; 43 44 orders = thp_supported_orders(); 45 46 for (i = 0; orders && count < max; i++) { 47 if (!(orders & (1UL << i))) 48 continue; 49 orders &= ~(1UL << i); 50 kb = (pagesize >> 10) << i; 51 sizes[count++] = kb * 1024; 52 ksft_print_msg("[INFO] detected THP size: %zu KiB\n", kb); 53 } 54 55 return count; 56 } 57 58 static void *mmap_aligned(size_t size, int prot, int flags) 59 { 60 size_t mmap_size = size * 2; 61 char *mmap_mem, *mem; 62 63 mmap_mem = mmap(NULL, mmap_size, prot, flags, -1, 0); 64 if (mmap_mem == MAP_FAILED) 65 return mmap_mem; 66 67 mem = (char *)(((uintptr_t)mmap_mem + size - 1) & ~(size - 1)); 68 munmap(mmap_mem, mem - mmap_mem); 69 munmap(mem + size, mmap_mem + mmap_size - mem - size); 70 71 return mem; 72 } 73 74 static void *alloc_one_folio(size_t size, bool private, bool hugetlb) 75 { 76 bool thp = !hugetlb && size > pagesize; 77 int flags = MAP_ANONYMOUS; 78 int prot = PROT_READ | PROT_WRITE; 79 char *mem, *addr; 80 81 assert((size & (size - 1)) == 0); 82 83 if (private) 84 flags |= MAP_PRIVATE; 85 else 86 flags |= MAP_SHARED; 87 88 /* 89 * For THP, we must explicitly enable the THP size, allocate twice the 90 * required space then manually align. 91 */ 92 if (thp) { 93 struct thp_settings settings = *thp_current_settings(); 94 95 if (private) 96 settings.hugepages[sz2ord(size, pagesize)].enabled = THP_ALWAYS; 97 else 98 settings.shmem_hugepages[sz2ord(size, pagesize)].enabled = SHMEM_ALWAYS; 99 100 thp_push_settings(&settings); 101 102 mem = mmap_aligned(size, prot, flags); 103 } else { 104 if (hugetlb) { 105 flags |= MAP_HUGETLB; 106 flags |= __builtin_ctzll(size) << MAP_HUGE_SHIFT; 107 } 108 109 mem = mmap(NULL, size, prot, flags, -1, 0); 110 } 111 112 if (mem == MAP_FAILED) { 113 mem = NULL; 114 goto out; 115 } 116 117 assert(((uintptr_t)mem & (size - 1)) == 0); 118 119 /* 120 * Populate the folio by writing the first byte and check that all pages 121 * are populated. Finally set the whole thing to non-zero data to avoid 122 * kernel from mapping it back to the zero page. 123 */ 124 mem[0] = 1; 125 for (addr = mem; addr < mem + size; addr += pagesize) { 126 if (!pagemap_is_populated(pagemap_fd, addr)) { 127 munmap(mem, size); 128 mem = NULL; 129 goto out; 130 } 131 } 132 memset(mem, 1, size); 133 out: 134 if (thp) 135 thp_pop_settings(); 136 137 return mem; 138 } 139 140 static bool check_uffd_wp_state(void *mem, size_t size, bool expect) 141 { 142 uint64_t pte; 143 void *addr; 144 145 for (addr = mem; addr < mem + size; addr += pagesize) { 146 pte = pagemap_get_entry(pagemap_fd, addr); 147 if (!!(pte & PM_UFFD_WP) != expect) { 148 ksft_test_result_fail("uffd-wp not %s for pte %lu!\n", 149 expect ? "set" : "clear", 150 (addr - mem) / pagesize); 151 return false; 152 } 153 } 154 155 return true; 156 } 157 158 static bool range_is_swapped(void *addr, size_t size) 159 { 160 for (; size; addr += pagesize, size -= pagesize) 161 if (!pagemap_is_swapped(pagemap_fd, addr)) 162 return false; 163 return true; 164 } 165 166 static void test_one_folio(uffd_global_test_opts_t *gopts, size_t size, bool private, 167 bool swapout, bool hugetlb) 168 { 169 struct uffdio_writeprotect wp_prms; 170 uint64_t features = 0; 171 void *addr = NULL; 172 void *mem = NULL; 173 174 assert(!(hugetlb && swapout)); 175 176 ksft_print_msg("[RUN] %s(size=%zu, private=%s, swapout=%s, hugetlb=%s)\n", 177 __func__, 178 size, 179 private ? "true" : "false", 180 swapout ? "true" : "false", 181 hugetlb ? "true" : "false"); 182 183 /* Allocate a folio of required size and type. */ 184 mem = alloc_one_folio(size, private, hugetlb); 185 if (!mem) { 186 ksft_test_result_fail("alloc_one_folio() failed\n"); 187 goto out; 188 } 189 190 /* Register range for uffd-wp. */ 191 if (userfaultfd_open(gopts, &features)) { 192 if (errno == ENOENT) 193 ksft_test_result_skip("userfaultfd not available\n"); 194 else 195 ksft_test_result_fail("userfaultfd_open() failed\n"); 196 goto out; 197 } 198 if (uffd_register(gopts->uffd, mem, size, false, true, false)) { 199 ksft_test_result_fail("uffd_register() failed\n"); 200 goto out; 201 } 202 wp_prms.mode = UFFDIO_WRITEPROTECT_MODE_WP; 203 wp_prms.range.start = (uintptr_t)mem; 204 wp_prms.range.len = size; 205 if (ioctl(gopts->uffd, UFFDIO_WRITEPROTECT, &wp_prms)) { 206 ksft_test_result_fail("ioctl(UFFDIO_WRITEPROTECT) failed\n"); 207 goto out; 208 } 209 210 if (swapout) { 211 madvise(mem, size, MADV_PAGEOUT); 212 if (!range_is_swapped(mem, size)) { 213 ksft_test_result_skip("MADV_PAGEOUT did not work, is swap enabled?\n"); 214 goto out; 215 } 216 } 217 218 /* Check that uffd-wp is set for all PTEs in range. */ 219 if (!check_uffd_wp_state(mem, size, true)) 220 goto out; 221 222 /* 223 * Move the mapping to a new, aligned location. Since 224 * UFFD_FEATURE_EVENT_REMAP is not set, we expect the uffd-wp bit for 225 * each PTE to be cleared in the new mapping. 226 */ 227 addr = mmap_aligned(size, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS); 228 if (addr == MAP_FAILED) { 229 ksft_test_result_fail("mmap_aligned() failed\n"); 230 goto out; 231 } 232 if (mremap(mem, size, size, MREMAP_FIXED | MREMAP_MAYMOVE, addr) == MAP_FAILED) { 233 ksft_test_result_fail("mremap() failed\n"); 234 munmap(addr, size); 235 goto out; 236 } 237 mem = addr; 238 239 /* Check that uffd-wp is cleared for all PTEs in range. */ 240 if (!check_uffd_wp_state(mem, size, false)) 241 goto out; 242 243 ksft_test_result_pass("%s(size=%zu, private=%s, swapout=%s, hugetlb=%s)\n", 244 __func__, 245 size, 246 private ? "true" : "false", 247 swapout ? "true" : "false", 248 hugetlb ? "true" : "false"); 249 out: 250 if (mem) 251 munmap(mem, size); 252 if (gopts->uffd >= 0) { 253 close(gopts->uffd); 254 gopts->uffd = -1; 255 } 256 } 257 258 struct testcase { 259 unsigned long *sizes; 260 int *nr_sizes; 261 bool private; 262 bool swapout; 263 bool hugetlb; 264 }; 265 266 static const struct testcase testcases[] = { 267 /* base pages. */ 268 { 269 .sizes = &pagesize, 270 .nr_sizes = &nr_pagesizes, 271 .private = false, 272 .swapout = false, 273 .hugetlb = false, 274 }, 275 { 276 .sizes = &pagesize, 277 .nr_sizes = &nr_pagesizes, 278 .private = true, 279 .swapout = false, 280 .hugetlb = false, 281 }, 282 { 283 .sizes = &pagesize, 284 .nr_sizes = &nr_pagesizes, 285 .private = false, 286 .swapout = true, 287 .hugetlb = false, 288 }, 289 { 290 .sizes = &pagesize, 291 .nr_sizes = &nr_pagesizes, 292 .private = true, 293 .swapout = true, 294 .hugetlb = false, 295 }, 296 297 /* thp. */ 298 { 299 .sizes = thpsizes, 300 .nr_sizes = &nr_thpsizes, 301 .private = false, 302 .swapout = false, 303 .hugetlb = false, 304 }, 305 { 306 .sizes = thpsizes, 307 .nr_sizes = &nr_thpsizes, 308 .private = true, 309 .swapout = false, 310 .hugetlb = false, 311 }, 312 { 313 .sizes = thpsizes, 314 .nr_sizes = &nr_thpsizes, 315 .private = false, 316 .swapout = true, 317 .hugetlb = false, 318 }, 319 { 320 .sizes = thpsizes, 321 .nr_sizes = &nr_thpsizes, 322 .private = true, 323 .swapout = true, 324 .hugetlb = false, 325 }, 326 327 /* hugetlb. */ 328 { 329 .sizes = hugetlbsizes, 330 .nr_sizes = &nr_hugetlbsizes, 331 .private = false, 332 .swapout = false, 333 .hugetlb = true, 334 }, 335 { 336 .sizes = hugetlbsizes, 337 .nr_sizes = &nr_hugetlbsizes, 338 .private = true, 339 .swapout = false, 340 .hugetlb = true, 341 }, 342 }; 343 344 int main(int argc, char **argv) 345 { 346 uffd_global_test_opts_t gopts = { 0 }; 347 struct thp_settings settings; 348 int i, j, plan = 0; 349 350 hugepage_save_settings(true, true); 351 352 check_uffd_wp_feature_supported(); 353 354 pagesize = getpagesize(); 355 nr_thpsizes = detect_thp_sizes(thpsizes, ARRAY_SIZE(thpsizes)); 356 nr_hugetlbsizes = hugetlb_setup(1, hugetlbsizes, ARRAY_SIZE(hugetlbsizes)); 357 358 /* If THP is supported, initially disable THP. */ 359 if (nr_thpsizes) { 360 thp_read_settings(&settings); 361 for (i = 0; i < NR_ORDERS; i++) { 362 settings.hugepages[i].enabled = THP_NEVER; 363 settings.shmem_hugepages[i].enabled = SHMEM_NEVER; 364 } 365 thp_push_settings(&settings); 366 } 367 368 for (i = 0; i < ARRAY_SIZE(testcases); i++) 369 plan += *testcases[i].nr_sizes; 370 ksft_set_plan(plan); 371 372 pagemap_fd = open("/proc/self/pagemap", O_RDONLY); 373 if (pagemap_fd < 0) 374 ksft_exit_fail_msg("opening pagemap failed\n"); 375 376 for (i = 0; i < ARRAY_SIZE(testcases); i++) { 377 const struct testcase *tc = &testcases[i]; 378 379 for (j = 0; j < *tc->nr_sizes; j++) 380 test_one_folio(&gopts, tc->sizes[j], tc->private, 381 tc->swapout, tc->hugetlb); 382 } 383 384 i = ksft_get_fail_cnt(); 385 if (i) 386 ksft_exit_fail_msg("%d out of %d tests failed\n", 387 i, ksft_test_num()); 388 ksft_exit_pass(); 389 } 390