1 #define _GNU_SOURCE 2 #include <ctype.h> 3 #include <errno.h> 4 #include <fcntl.h> 5 #include <limits.h> 6 #include <dirent.h> 7 #include <signal.h> 8 #include <stdio.h> 9 #include <stdlib.h> 10 #include <stdbool.h> 11 #include <string.h> 12 #include <unistd.h> 13 14 #include <linux/mman.h> 15 #include <sys/mman.h> 16 #include <sys/wait.h> 17 #include <sys/types.h> 18 #include <sys/stat.h> 19 #include <sys/sysmacros.h> 20 #include <sys/vfs.h> 21 22 #include "linux/magic.h" 23 24 #include "vm_util.h" 25 26 #define BASE_ADDR ((void *)(1UL << 30)) 27 static unsigned long hpage_pmd_size; 28 static int hpage_pmd_order; 29 static unsigned long page_size; 30 static int hpage_pmd_nr; 31 static int anon_order; 32 static int collapse_order; 33 static bool collapse_order_set; 34 static int collapse_orders[NR_ORDERS]; 35 static int nr_collapse_orders; 36 static int pagemap_fd = -1; 37 static int kpageflags_fd = -1; 38 39 #define PID_SMAPS "/proc/self/smaps" 40 #define TEST_FILE "collapse_test_file" 41 42 #define MAX_LINE_LENGTH 500 43 44 enum vma_type { 45 VMA_ANON, 46 VMA_FILE, 47 VMA_SHMEM, 48 }; 49 50 enum file_setup_ops { 51 FILE_SETUP_READ_ONLY_FS, 52 FILE_SETUP_READ_WRITE_FS_READ_DATA, 53 FILE_SETUP_READ_WRITE_FS_WRITE_DATA, 54 }; 55 56 struct mem_ops { 57 void *(*setup_area)(int nr_hpages); 58 void (*cleanup_area)(void *p, unsigned long size); 59 void (*fault)(void *p, unsigned long start, unsigned long end); 60 bool (*check_huge)(void *addr, size_t len, int nr_hpages, uint64_t hpage_size); 61 const char *name; 62 }; 63 64 static struct mem_ops *read_only_file_ops; 65 static struct mem_ops *read_write_file_read_ops; 66 static struct mem_ops *read_write_file_write_ops; 67 static struct mem_ops *anon_ops; 68 static struct mem_ops *shmem_ops; 69 70 struct collapse_context { 71 void (*collapse)(const char *msg, char *p, int nr_hpages, 72 struct mem_ops *ops, bool expect); 73 bool enforce_pte_scan_limits; 74 const char *name; 75 }; 76 77 static struct collapse_context *khugepaged_context; 78 static struct collapse_context *mthp_khugepaged_context; 79 static struct collapse_context *madvise_context; 80 81 struct file_info { 82 const char *dir; 83 char path[PATH_MAX]; 84 enum vma_type type; 85 int fd; 86 char dev_queue_read_ahead_path[PATH_MAX]; 87 }; 88 89 static struct file_info finfo; 90 static int exit_status; 91 92 static void success(const char *msg) 93 { 94 printf(" \e[32m%s\e[0m\n", msg); 95 exit_status = KSFT_PASS; 96 } 97 98 static void fail(const char *msg) 99 { 100 printf(" \e[31m%s\e[0m\n", msg); 101 exit_status = KSFT_FAIL; 102 } 103 104 static void skip(const char *msg) 105 { 106 printf(" \e[33m%s\e[0m\n", msg); 107 exit_status = KSFT_SKIP; 108 } 109 110 static void save_settings(void) 111 { 112 ksft_print_msg("Save THP and khugepaged settings..."); 113 if ((read_only_file_ops || read_write_file_read_ops || 114 read_write_file_write_ops) && 115 finfo.type == VMA_FILE) 116 thp_set_read_ahead_path(finfo.dev_queue_read_ahead_path); 117 thp_save_settings(); 118 119 success("OK"); 120 } 121 122 static void get_finfo(const char *dir) 123 { 124 struct stat path_stat; 125 struct statfs fs; 126 char buf[1 << 10]; 127 char path[PATH_MAX]; 128 char *str, *end; 129 int ret; 130 131 finfo.dir = dir; 132 if (stat(finfo.dir, &path_stat)) 133 ksft_exit_fail_perror("stat()"); 134 if (!S_ISDIR(path_stat.st_mode)) 135 ksft_exit_fail_msg("%s: Not a directory (%s)\n", __func__, finfo.dir); 136 if (snprintf(finfo.path, sizeof(finfo.path), "%s/" TEST_FILE, 137 finfo.dir) >= sizeof(finfo.path)) 138 ksft_exit_fail_msg("%s: Pathname is too long\n", __func__); 139 if (statfs(finfo.dir, &fs)) 140 ksft_exit_fail_perror("statfs()"); 141 finfo.type = fs.f_type == TMPFS_MAGIC ? VMA_SHMEM : VMA_FILE; 142 if (finfo.type == VMA_SHMEM) 143 return; 144 145 /* Find owning device's queue/read_ahead_kb control */ 146 if (snprintf(path, sizeof(path), "/sys/dev/block/%d:%d/uevent", 147 major(path_stat.st_dev), minor(path_stat.st_dev)) 148 >= sizeof(path)) 149 ksft_exit_fail_msg("%s: Pathname is too long\n", __func__); 150 ret = read_file(path, buf, sizeof(buf)); 151 if (ret) 152 ksft_exit_fail_msg("read_file(%s): %s\n", path, strerror(-ret)); 153 if (strstr(buf, "DEVTYPE=disk")) { 154 /* Found it */ 155 if (snprintf(finfo.dev_queue_read_ahead_path, 156 sizeof(finfo.dev_queue_read_ahead_path), 157 "/sys/dev/block/%d:%d/queue/read_ahead_kb", 158 major(path_stat.st_dev), minor(path_stat.st_dev)) 159 >= sizeof(finfo.dev_queue_read_ahead_path)) 160 ksft_exit_fail_msg("%s: Pathname is too long\n", __func__); 161 return; 162 } 163 if (!strstr(buf, "DEVTYPE=partition")) 164 ksft_exit_fail_msg("%s: Unknown device type: %s\n", __func__, path); 165 /* 166 * Partition of block device - need to find actual device. 167 * Using naming convention that devnameN is partition of 168 * device devname. 169 */ 170 str = strstr(buf, "DEVNAME="); 171 if (!str) 172 ksft_exit_fail_msg("%s: Could not read: %s", __func__, path); 173 str += 8; 174 end = str; 175 while (*end) { 176 if (isdigit(*end)) { 177 *end = '\0'; 178 if (snprintf(finfo.dev_queue_read_ahead_path, 179 sizeof(finfo.dev_queue_read_ahead_path), 180 "/sys/block/%s/queue/read_ahead_kb", 181 str) >= sizeof(finfo.dev_queue_read_ahead_path)) 182 ksft_exit_fail_msg("%s: Pathname is too long\n", __func__); 183 return; 184 } 185 ++end; 186 } 187 ksft_exit_fail_msg("%s: Could not read: %s\n", __func__, path); 188 } 189 190 static bool check_swap(void *addr, unsigned long size) 191 { 192 bool swap = false; 193 int ret; 194 FILE *fp; 195 char buffer[MAX_LINE_LENGTH]; 196 char addr_pattern[MAX_LINE_LENGTH]; 197 198 ret = snprintf(addr_pattern, MAX_LINE_LENGTH, "%08lx-", 199 (unsigned long) addr); 200 if (ret >= MAX_LINE_LENGTH) 201 ksft_exit_fail_msg("%s: Pattern is too long\n", __func__); 202 203 fp = fopen(PID_SMAPS, "r"); 204 if (!fp) 205 ksft_exit_fail_msg("%s: Failed to open file %s\n", __func__, PID_SMAPS); 206 if (!check_for_pattern(fp, addr_pattern, buffer, sizeof(buffer))) 207 goto err_out; 208 209 ret = snprintf(addr_pattern, MAX_LINE_LENGTH, "Swap:%19ld kB", 210 size >> 10); 211 if (ret >= MAX_LINE_LENGTH) 212 ksft_exit_fail_msg("%s: Pattern is too long\n", __func__); 213 /* 214 * Fetch the Swap: in the same block and check whether it got 215 * the expected number of hugeepages next. 216 */ 217 if (!check_for_pattern(fp, "Swap:", buffer, sizeof(buffer))) 218 goto err_out; 219 220 if (strncmp(buffer, addr_pattern, strlen(addr_pattern))) 221 goto err_out; 222 223 swap = true; 224 err_out: 225 fclose(fp); 226 return swap; 227 } 228 229 static bool swapout_range(void *p, unsigned long size) 230 { 231 int i; 232 233 /* keep khugepaged from collapsing the range and swapping it back in */ 234 if (madvise(p, size, MADV_NOHUGEPAGE)) 235 ksft_exit_fail_perror("madvise(MADV_NOHUGEPAGE)"); 236 237 /* 238 * Retry several times because MADV_PAGEOUT is best effort. Sleep 239 * between the retries to give outstanding writeback a chance to 240 * finish. 241 */ 242 for (i = 0; i < 40; i++) { 243 if (madvise(p, size, MADV_PAGEOUT)) 244 ksft_exit_fail_perror("madvise(MADV_PAGEOUT)"); 245 if (check_swap(p, size)) 246 return true; 247 usleep(50 * 1000); 248 } 249 return false; 250 } 251 252 static void *alloc_mapping(int nr) 253 { 254 void *p; 255 256 p = mmap(BASE_ADDR, nr * hpage_pmd_size, PROT_READ | PROT_WRITE, 257 MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); 258 if (p != BASE_ADDR) 259 ksft_exit_fail_msg("Failed to allocate VMA at %p\n", BASE_ADDR); 260 261 return p; 262 } 263 264 static void fill_memory(int *p, unsigned long start, unsigned long end) 265 { 266 int i; 267 268 for (i = start / page_size; i < end / page_size; i++) 269 p[i * page_size / sizeof(*p)] = i + 0xdead0000; 270 } 271 272 /* 273 * MADV_COLLAPSE is a best-effort request and may fail if an internal 274 * resource is temporarily unavailable, in which case it will set errno to 275 * EAGAIN. In such a case, immediately reattempt the operation one more 276 * time. 277 */ 278 static int madvise_collapse_retry(void *p, unsigned long size) 279 { 280 bool retry = true; 281 int ret; 282 283 retry: 284 ret = madvise(p, size, MADV_COLLAPSE); 285 if (ret && errno == EAGAIN && retry) { 286 retry = false; 287 goto retry; 288 } 289 return ret; 290 } 291 292 /* 293 * Returns pmd-mapped hugepage in VMA marked VM_HUGEPAGE, filled with 294 * validate_memory()'able contents. 295 */ 296 static void *alloc_hpage(struct mem_ops *ops) 297 { 298 void *p = ops->setup_area(1); 299 300 ops->fault(p, 0, hpage_pmd_size); 301 302 /* 303 * VMA should be neither VM_HUGEPAGE nor VM_NOHUGEPAGE. 304 * The latter is ineligible for collapse by MADV_COLLAPSE 305 * while the former might cause MADV_COLLAPSE to race with 306 * khugepaged on low-load system (like a test machine), which 307 * would cause MADV_COLLAPSE to fail with EAGAIN. 308 */ 309 ksft_print_msg("Allocate huge page..."); 310 if (madvise_collapse_retry(p, hpage_pmd_size)) 311 ksft_exit_fail_perror("madvise(MADV_COLLAPSE)"); 312 if (!ops->check_huge(p, hpage_pmd_size, 1, hpage_pmd_size)) 313 ksft_exit_fail_perror("madvise(MADV_COLLAPSE)"); 314 if (madvise(p, hpage_pmd_size, MADV_HUGEPAGE)) 315 ksft_exit_fail_perror("madvise(MADV_HUGEPAGE)"); 316 success("OK"); 317 return p; 318 } 319 320 static void validate_memory(int *p, unsigned long start, unsigned long end) 321 { 322 int i; 323 324 for (i = start / page_size; i < end / page_size; i++) { 325 if (p[i * page_size / sizeof(*p)] != i + 0xdead0000) 326 ksft_exit_fail_msg("Page %d is corrupted: %#x\n", 327 i, p[i * page_size / sizeof(*p)]); 328 } 329 } 330 331 static void *anon_setup_area(int nr_hpages) 332 { 333 return alloc_mapping(nr_hpages); 334 } 335 336 static void anon_cleanup_area(void *p, unsigned long size) 337 { 338 munmap(p, size); 339 } 340 341 static void anon_fault(void *p, unsigned long start, unsigned long end) 342 { 343 fill_memory(p, start, end); 344 } 345 346 static void *file_setup_area_common(int nr_hpages, enum file_setup_ops setup) 347 { 348 const int open_opt = setup == FILE_SETUP_READ_ONLY_FS ? O_RDONLY : O_RDWR; 349 const int mmap_prot = setup == FILE_SETUP_READ_ONLY_FS ? PROT_READ : (PROT_READ | PROT_WRITE); 350 int fd, ret; 351 void *p; 352 unsigned long size; 353 354 unlink(finfo.path); /* Cleanup from previous failed tests */ 355 ksft_print_msg("Creating %s for collapse%s...", finfo.path, 356 finfo.type == VMA_SHMEM ? " (tmpfs)" : ""); 357 fd = open(finfo.path, O_CREAT | O_RDWR | O_TRUNC | O_EXCL, 358 777); 359 if (fd < 0) 360 ksft_exit_fail_perror("open()"); 361 362 size = nr_hpages * hpage_pmd_size; 363 if (ftruncate(fd, size)) 364 ksft_exit_fail_perror("ftruncate()"); 365 p = mmap(BASE_ADDR, size, PROT_READ | PROT_WRITE, 366 MAP_SHARED, fd, 0); 367 if (p != BASE_ADDR) 368 ksft_exit_fail_perror("mmap()"); 369 fill_memory(p, 0, size); 370 if (msync(p, size, MS_SYNC)) 371 ksft_exit_fail_perror("msync()"); 372 close(fd); 373 munmap(p, size); 374 success("OK"); 375 ksft_print_msg("Opening %s %s for collapse...", finfo.path, 376 setup == FILE_SETUP_READ_ONLY_FS ? "read-only" : 377 setup == FILE_SETUP_READ_WRITE_FS_READ_DATA ? 378 "read-write (read)" : 379 "read-write (write)"); 380 finfo.fd = open(finfo.path, open_opt, 777); 381 if (finfo.fd < 0) 382 ksft_exit_fail_perror("open()"); 383 p = mmap(BASE_ADDR, size, mmap_prot, MAP_SHARED, finfo.fd, 0); 384 if (p == MAP_FAILED || p != BASE_ADDR) 385 ksft_exit_fail_perror("mmap()"); 386 387 /* Drop page cache */ 388 ret = write_file("/proc/sys/vm/drop_caches", "3", 2); 389 if (ret) 390 ksft_exit_fail_msg("write_file(drop_caches): %s\n", 391 strerror(-ret)); 392 393 success("OK"); 394 return p; 395 } 396 397 static void *file_setup_read_only_area(int nr_hpages) 398 { 399 return file_setup_area_common(nr_hpages, FILE_SETUP_READ_ONLY_FS); 400 } 401 402 static void *file_setup_read_write_fs_read_area(int nr_hpages) 403 { 404 return file_setup_area_common(nr_hpages, FILE_SETUP_READ_WRITE_FS_READ_DATA); 405 } 406 407 static void *file_setup_read_write_fs_write_area(int nr_hpages) 408 { 409 return file_setup_area_common(nr_hpages, FILE_SETUP_READ_WRITE_FS_WRITE_DATA); 410 } 411 412 static void file_cleanup_area(void *p, unsigned long size) 413 { 414 munmap(p, size); 415 close(finfo.fd); 416 unlink(finfo.path); 417 } 418 419 static void file_fault_read(void *p, unsigned long start, unsigned long end) 420 { 421 if (madvise(((char *)p) + start, end - start, MADV_POPULATE_READ)) 422 ksft_exit_fail_perror("madvise(MADV_POPULATE_READ)"); 423 } 424 425 static void file_fault_read_and_flush(void *p, unsigned long start, unsigned long end) 426 { 427 file_fault_read(p, start, end); 428 /* 429 * make folio clean, since dirty folios from read&write file are 430 * rejected and not flushed 431 */ 432 msync((char *)p + start, end - start, MS_SYNC); 433 } 434 435 static void file_fault_write(void *p, unsigned long start, unsigned long end) 436 { 437 if (madvise(((char *)p) + start, end - start, MADV_POPULATE_WRITE)) 438 ksft_exit_fail_perror("madvise(MADV_POPULATE_WRITE)"); 439 } 440 441 static void *shmem_setup_area(int nr_hpages) 442 { 443 void *p; 444 unsigned long size = nr_hpages * hpage_pmd_size; 445 446 finfo.fd = memfd_create("khugepaged-selftest-collapse-shmem", 0); 447 if (finfo.fd < 0) 448 ksft_exit_fail_perror("memfd_create()"); 449 if (ftruncate(finfo.fd, size)) 450 ksft_exit_fail_perror("ftruncate()"); 451 p = mmap(BASE_ADDR, size, PROT_READ | PROT_WRITE, MAP_SHARED, finfo.fd, 452 0); 453 if (p != BASE_ADDR) 454 ksft_exit_fail_perror("mmap()"); 455 return p; 456 } 457 458 static void shmem_cleanup_area(void *p, unsigned long size) 459 { 460 munmap(p, size); 461 close(finfo.fd); 462 } 463 464 static struct mem_ops __anon_ops = { 465 .setup_area = &anon_setup_area, 466 .cleanup_area = &anon_cleanup_area, 467 .fault = &anon_fault, 468 .check_huge = &check_huge_anon, 469 .name = "anon", 470 }; 471 472 static struct mem_ops __read_only_file_ops = { 473 .setup_area = &file_setup_read_only_area, 474 .cleanup_area = &file_cleanup_area, 475 .fault = &file_fault_read, 476 .check_huge = &check_huge_file, 477 .name = "file", 478 }; 479 480 static struct mem_ops __read_write_file_read_ops = { 481 .setup_area = &file_setup_read_write_fs_read_area, 482 .cleanup_area = &file_cleanup_area, 483 .fault = &file_fault_read_and_flush, 484 .check_huge = &check_huge_file, 485 .name = "file", 486 }; 487 488 static struct mem_ops __read_write_file_write_ops = { 489 .setup_area = &file_setup_read_write_fs_write_area, 490 .cleanup_area = &file_cleanup_area, 491 .fault = &file_fault_write, 492 .check_huge = &check_huge_file, 493 .name = "file", 494 }; 495 496 static struct mem_ops __shmem_ops = { 497 .setup_area = &shmem_setup_area, 498 .cleanup_area = &shmem_cleanup_area, 499 .fault = &anon_fault, 500 .check_huge = &check_huge_file, 501 .name = "shmem", 502 }; 503 504 static bool is_tmpfs(struct mem_ops *ops) 505 { 506 return (ops == &__read_only_file_ops || 507 ops == &__read_write_file_read_ops || 508 ops == &__read_write_file_write_ops) && 509 finfo.type == VMA_SHMEM; 510 } 511 512 static bool is_anon(struct mem_ops *ops) 513 { 514 return ops == &__anon_ops; 515 } 516 517 static void __madvise_collapse(const char *msg, char *p, int nr_hpages, 518 struct mem_ops *ops, bool expect) 519 { 520 struct thp_settings settings = *thp_current_settings(); 521 int ret, i; 522 523 ksft_print_msg("%s...", msg); 524 525 /* 526 * read&write file collapse succeeds for MADV_COLLAPSE because dirty 527 * folios are written back after collapse fails for dirty folios and 528 * another collapse is attempted. 529 */ 530 531 /* 532 * Prevent khugepaged interference and tests that MADV_COLLAPSE 533 * ignores /sys/kernel/mm/transparent_hugepage/enabled 534 */ 535 settings.thp_enabled = THP_NEVER; 536 settings.shmem_enabled = SHMEM_NEVER; 537 for (i = 0; i < NR_ORDERS; i++) { 538 settings.hugepages[i].enabled = THP_NEVER; 539 settings.shmem_hugepages[i].enabled = SHMEM_NEVER; 540 } 541 thp_push_settings(&settings); 542 543 /* Clear VM_NOHUGEPAGE */ 544 madvise(p, nr_hpages * hpage_pmd_size, MADV_HUGEPAGE); 545 ret = madvise_collapse_retry(p, nr_hpages * hpage_pmd_size); 546 if (((bool)ret) == expect) 547 fail("Fail: Bad return value"); 548 else if (!ops->check_huge(p, nr_hpages * hpage_pmd_size, expect ? nr_hpages : 0, hpage_pmd_size)) 549 fail("Fail: check_huge()"); 550 else 551 success("OK"); 552 553 thp_pop_settings(); 554 } 555 556 static void madvise_collapse(const char *msg, char *p, int nr_hpages, 557 struct mem_ops *ops, bool expect) 558 { 559 /* Sanity check */ 560 if (!ops->check_huge(p, nr_hpages * hpage_pmd_size, 0, hpage_pmd_size)) 561 ksft_exit_fail_msg("Unexpected huge page\n"); 562 __madvise_collapse(msg, p, nr_hpages, ops, expect); 563 } 564 565 #define TICK 500000 566 static bool wait_for_scan(const char *msg, char *p, size_t len, 567 int nr_hpages, int collap_order, struct mem_ops *ops) 568 { 569 unsigned long hpage_size = page_size << collap_order; 570 unsigned long bytes = (unsigned long)nr_hpages * hpage_size; 571 int timeout, full_scans; 572 573 /* Half-second ticks: three seconds floor, plus a second per 128M */ 574 timeout = 6 + 2 * (bytes / (128UL << 20)); 575 576 /* Sanity check */ 577 if (!ops->check_huge(p, len, 0, hpage_size)) 578 ksft_exit_fail_msg("Unexpected huge page\n"); 579 580 madvise(p, len, MADV_HUGEPAGE); 581 582 /* Wait until the second full_scan completed */ 583 full_scans = thp_read_num("khugepaged/full_scans") + 2; 584 585 ksft_print_msg("%s...", msg); 586 while (timeout--) { 587 if (ops->check_huge(p, len, nr_hpages, hpage_size)) 588 break; 589 if (thp_read_num("khugepaged/full_scans") >= full_scans) 590 break; 591 printf("."); 592 usleep(TICK); 593 } 594 595 /* 596 * The file and shmem tests rely on refaults to install PMD mappings 597 * after collapse. MADV_NOHUGEPAGE would prevent those mappings. 598 * 599 * Apply MADV_NOHUGEPAGE only to anonymous VMAs to prevent khugepaged 600 * from unexpectedly collapsing pages during the test. 601 */ 602 if (is_anon(ops)) 603 madvise(p, len, MADV_NOHUGEPAGE); 604 605 return timeout == -1; 606 } 607 608 static void khugepaged_collapse(const char *msg, char *p, int nr_hpages, 609 struct mem_ops *ops, bool expect) 610 { 611 size_t len = nr_hpages * hpage_pmd_size; 612 613 /* 614 * read&write file collapse fails since khugepaged does not flush 615 * the target dirty folios 616 */ 617 if (!is_tmpfs(ops) && ops == &__read_write_file_write_ops) 618 expect = false; 619 620 if (wait_for_scan(msg, p, len, nr_hpages, hpage_pmd_order, ops)) { 621 if (expect) 622 fail("Timeout"); 623 else 624 success("OK"); 625 return; 626 } 627 628 /* 629 * For file and shmem memory, khugepaged only retracts pte entries after 630 * putting the new hugepage in the page cache. The hugepage must be 631 * subsequently refaulted to install the pmd mapping for the mm. 632 */ 633 if (ops != &__anon_ops) 634 ops->fault(p, 0, nr_hpages * hpage_pmd_size); 635 636 if (ops->check_huge(p, len, expect ? nr_hpages : 0, hpage_pmd_size)) 637 success("OK"); 638 else 639 fail("Fail"); 640 } 641 642 static void mthp_khugepaged_collapse(const char *msg, char *p, int nr_hpages, 643 struct mem_ops *ops, bool expect) 644 { 645 unsigned long hpage_size = page_size << collapse_order; 646 struct thp_settings settings = *thp_current_settings(); 647 /* mTHP collpase only allocates PMD sized memory */ 648 size_t len = hpage_pmd_size; 649 650 /* Set mTHP setting for mTHP collapse */ 651 if (ops == &__anon_ops) { 652 settings.thp_enabled = THP_NEVER; 653 settings.hugepages[collapse_order].enabled = THP_MADVISE; 654 } 655 656 thp_push_settings(&settings); 657 658 if (wait_for_scan(msg, p, len, nr_hpages, collapse_order, ops)) { 659 if (expect) 660 fail("Timeout"); 661 else 662 success("OK"); 663 664 /* Restore THP settings for mTHP collapse. */ 665 thp_pop_settings(); 666 return; 667 } 668 669 /* 670 * For file and shmem memory, khugepaged only retracts pte entries after 671 * putting the new hugepage in the page cache. The hugepage must be 672 * subsequently refaulted to install the pmd mapping for the mm. 673 */ 674 if (ops != &__anon_ops) 675 ops->fault(p, 0, nr_hpages * hpage_size); 676 677 if (ops->check_huge(p, len, expect ? nr_hpages : 0, hpage_size)) 678 success("OK"); 679 else 680 fail("Fail"); 681 682 /* Restore THP settings for mTHP collapse. */ 683 thp_pop_settings(); 684 } 685 686 static struct collapse_context __khugepaged_context = { 687 .collapse = &khugepaged_collapse, 688 .enforce_pte_scan_limits = true, 689 .name = "khugepaged", 690 }; 691 692 static struct collapse_context __mthp_khugepaged_context = { 693 .collapse = &mthp_khugepaged_collapse, 694 .enforce_pte_scan_limits = true, 695 .name = "mthp_khugepaged", 696 }; 697 698 static struct collapse_context __madvise_context = { 699 .collapse = &madvise_collapse, 700 .enforce_pte_scan_limits = false, 701 .name = "madvise", 702 }; 703 704 static void alloc_at_fault(void) 705 { 706 struct thp_settings settings = *thp_current_settings(); 707 char *p; 708 709 settings.thp_enabled = THP_ALWAYS; 710 thp_push_settings(&settings); 711 712 p = alloc_mapping(1); 713 *p = 1; 714 ksft_print_msg("Allocate huge page on fault..."); 715 if (check_huge_anon(p, hpage_pmd_size, 1, hpage_pmd_size)) 716 success("OK"); 717 else 718 fail("Fail"); 719 720 thp_pop_settings(); 721 722 madvise(p, page_size, MADV_DONTNEED); 723 ksft_print_msg("Split huge PMD on MADV_DONTNEED..."); 724 if (check_huge_anon(p, hpage_pmd_size, 0, hpage_pmd_size)) 725 success("OK"); 726 else 727 fail("Fail"); 728 munmap(p, hpage_pmd_size); 729 730 ksft_test_result_report(exit_status, "allocate on fault and split\n"); 731 } 732 733 static void collapse_full(struct collapse_context *c, struct mem_ops *ops) 734 { 735 void *p; 736 int nr_pmds = 4, nr_hpages = 4; 737 unsigned long size = nr_hpages * hpage_pmd_size; 738 739 /* Only try 1 PMD sized range for mTHP collapse. */ 740 if (c == &__mthp_khugepaged_context) { 741 nr_pmds = 1; 742 nr_hpages = 1 << (hpage_pmd_order - collapse_order); 743 size = hpage_pmd_size; 744 } 745 746 p = ops->setup_area(nr_pmds); 747 ops->fault(p, 0, size); 748 c->collapse("Collapse multiple fully populated PTE table", p, nr_hpages, 749 ops, true); 750 validate_memory(p, 0, size); 751 ops->cleanup_area(p, size); 752 753 ksft_test_result_report(exit_status, "%s\n", __func__); 754 } 755 756 static void collapse_empty(struct collapse_context *c, struct mem_ops *ops) 757 { 758 int nr_hpages = 1; 759 void *p; 760 761 if (c == &__mthp_khugepaged_context) 762 nr_hpages = 1 << (hpage_pmd_order - collapse_order); 763 764 p = ops->setup_area(1); 765 c->collapse("Do not collapse empty PTE table", p, nr_hpages, ops, false); 766 ops->cleanup_area(p, hpage_pmd_size); 767 ksft_test_result_report(exit_status, "%s\n", __func__); 768 } 769 770 static void collapse_single_mthp(struct collapse_context *c, struct mem_ops *ops) 771 { 772 unsigned long hpage_size = page_size << collapse_order; 773 void *p; 774 775 p = ops->setup_area(1); 776 /* 777 * Only fault collapse_order sized ranges, and only check 1 778 * collapse_order sized huge page. 779 */ 780 ops->fault(p, 0, hpage_size); 781 c->collapse("Collapse PTE table with half PTE entries present", 782 p, 1, ops, true); 783 ops->cleanup_area(p, hpage_pmd_size); 784 ksft_test_result_report(exit_status, "%s\n", __func__); 785 } 786 787 static void collapse_single_pte_entry(struct collapse_context *c, struct mem_ops *ops) 788 { 789 void *p; 790 791 p = ops->setup_area(1); 792 ops->fault(p, 0, page_size); 793 c->collapse("Collapse PTE table with single PTE entry present", p, 794 1, ops, true); 795 ops->cleanup_area(p, hpage_pmd_size); 796 ksft_test_result_report(exit_status, "%s\n", __func__); 797 } 798 799 static void collapse_max_ptes_none(struct collapse_context *c, struct mem_ops *ops) 800 { 801 int max_ptes_none = hpage_pmd_nr / 2; 802 struct thp_settings settings = *thp_current_settings(); 803 void *p; 804 int fault_nr_pages = is_anon(ops) ? 1 << anon_order : 1; 805 806 settings.khugepaged.max_ptes_none = max_ptes_none; 807 thp_push_settings(&settings); 808 809 p = ops->setup_area(1); 810 811 if (is_tmpfs(ops)) { 812 /* shmem pages always in the page cache */ 813 printf("tmpfs..."); 814 skip("Skip"); 815 goto skip; 816 } 817 818 ops->fault(p, 0, (hpage_pmd_nr - max_ptes_none - fault_nr_pages) * page_size); 819 c->collapse("Maybe collapse with max_ptes_none exceeded", p, 1, 820 ops, !c->enforce_pte_scan_limits); 821 validate_memory(p, 0, (hpage_pmd_nr - max_ptes_none - fault_nr_pages) * page_size); 822 823 if (c->enforce_pte_scan_limits) { 824 ops->cleanup_area(p, hpage_pmd_size); 825 p = ops->setup_area(1); 826 827 ops->fault(p, 0, (hpage_pmd_nr - max_ptes_none) * page_size); 828 c->collapse("Collapse with max_ptes_none PTEs empty", p, 1, ops, 829 true); 830 validate_memory(p, 0, 831 (hpage_pmd_nr - max_ptes_none) * page_size); 832 } 833 skip: 834 ops->cleanup_area(p, hpage_pmd_size); 835 thp_pop_settings(); 836 ksft_test_result_report(exit_status, "%s\n", __func__); 837 } 838 839 static void collapse_swapin_single_pte(struct collapse_context *c, struct mem_ops *ops) 840 { 841 void *p; 842 843 p = ops->setup_area(1); 844 ops->fault(p, 0, hpage_pmd_size); 845 846 ksft_print_msg("Swapout one page..."); 847 if (swapout_range(p, page_size)) { 848 success("OK"); 849 } else { 850 skip("Could not swap out"); 851 goto out; 852 } 853 854 c->collapse("Collapse with swapping in single PTE entry", p, 1, ops, 855 true); 856 validate_memory(p, 0, hpage_pmd_size); 857 out: 858 ops->cleanup_area(p, hpage_pmd_size); 859 ksft_test_result_report(exit_status, "%s\n", __func__); 860 } 861 862 static void collapse_max_ptes_swap(struct collapse_context *c, struct mem_ops *ops) 863 { 864 int max_ptes_swap = thp_read_num("khugepaged/max_ptes_swap"); 865 void *p; 866 867 p = ops->setup_area(1); 868 ops->fault(p, 0, hpage_pmd_size); 869 870 ksft_print_msg("Swapout %d of %d pages...", max_ptes_swap + 1, hpage_pmd_nr); 871 if (swapout_range(p, (max_ptes_swap + 1) * page_size)) { 872 success("OK"); 873 } else { 874 skip("Could not swap out"); 875 goto out; 876 } 877 878 c->collapse("Maybe collapse with max_ptes_swap exceeded", p, 1, ops, 879 !c->enforce_pte_scan_limits); 880 validate_memory(p, 0, hpage_pmd_size); 881 882 if (c->enforce_pte_scan_limits) { 883 ops->fault(p, 0, hpage_pmd_size); 884 ksft_print_msg("Swapout %d of %d pages...", max_ptes_swap, 885 hpage_pmd_nr); 886 if (swapout_range(p, max_ptes_swap * page_size)) { 887 success("OK"); 888 } else { 889 skip("Could not swap out"); 890 goto out; 891 } 892 893 c->collapse("Collapse with max_ptes_swap pages swapped out", p, 894 1, ops, true); 895 validate_memory(p, 0, hpage_pmd_size); 896 } 897 out: 898 ops->cleanup_area(p, hpage_pmd_size); 899 ksft_test_result_report(exit_status, "%s\n", __func__); 900 } 901 902 static void collapse_single_pte_entry_compound(struct collapse_context *c, struct mem_ops *ops) 903 { 904 void *p; 905 906 p = alloc_hpage(ops); 907 908 if (is_tmpfs(ops)) { 909 /* MADV_DONTNEED won't evict tmpfs pages */ 910 printf("tmpfs..."); 911 skip("Skip"); 912 goto skip; 913 } 914 915 madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE); 916 ksft_print_msg("Split huge page leaving single PTE mapping compound page..."); 917 madvise(p + page_size, hpage_pmd_size - page_size, MADV_DONTNEED); 918 if (ops->check_huge(p, hpage_pmd_size, 0, hpage_pmd_size)) 919 success("OK"); 920 else 921 fail("Fail"); 922 923 c->collapse("Collapse PTE table with single PTE mapping compound page", 924 p, 1, ops, true); 925 validate_memory(p, 0, page_size); 926 skip: 927 ops->cleanup_area(p, hpage_pmd_size); 928 ksft_test_result_report(exit_status, "%s\n", __func__); 929 } 930 931 static void collapse_full_of_compound(struct collapse_context *c, struct mem_ops *ops) 932 { 933 void *p; 934 935 p = alloc_hpage(ops); 936 ksft_print_msg("Split huge page leaving single PTE page table full of compound pages..."); 937 madvise(p, page_size, MADV_NOHUGEPAGE); 938 madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE); 939 if (ops->check_huge(p, hpage_pmd_size, 0, hpage_pmd_size)) 940 success("OK"); 941 else 942 fail("Fail"); 943 944 c->collapse("Collapse PTE table full of compound pages", p, 1, ops, 945 true); 946 validate_memory(p, 0, hpage_pmd_size); 947 ops->cleanup_area(p, hpage_pmd_size); 948 ksft_test_result_report(exit_status, "%s\n", __func__); 949 } 950 951 static void collapse_compound_extreme(struct collapse_context *c, struct mem_ops *ops) 952 { 953 void *p; 954 int i; 955 956 /* 957 * This needs hpage_pmd_nr PMD-order allocations in a row, which the 958 * allocator will not supply if the PMD is very large. 959 */ 960 if (hpage_pmd_size > (32UL << 20)) { 961 ksft_test_result_skip("%s: PMD too large for fault-time THP construction\n", 962 __func__); 963 return; 964 } 965 966 p = ops->setup_area(1); 967 ksft_print_msg("Construct PTE page table full of different PTE-mapped compound pages\n"); 968 for (i = 0; i < hpage_pmd_nr; i++) { 969 madvise(BASE_ADDR, hpage_pmd_size, MADV_HUGEPAGE); 970 ops->fault(BASE_ADDR, 0, hpage_pmd_size); 971 if (!ops->check_huge(BASE_ADDR, hpage_pmd_size, 1, hpage_pmd_size)) 972 ksft_exit_fail_msg("Failed to allocate huge page\n"); 973 madvise(BASE_ADDR, hpage_pmd_size, MADV_NOHUGEPAGE); 974 975 p = mremap(BASE_ADDR - i * page_size, 976 i * page_size + hpage_pmd_size, 977 (i + 1) * page_size, 978 MREMAP_MAYMOVE | MREMAP_FIXED, 979 BASE_ADDR + 2 * hpage_pmd_size); 980 if (p == MAP_FAILED) 981 ksft_exit_fail_perror("mremap+unmap"); 982 983 p = mremap(BASE_ADDR + 2 * hpage_pmd_size, 984 (i + 1) * page_size, 985 (i + 1) * page_size + hpage_pmd_size, 986 MREMAP_MAYMOVE | MREMAP_FIXED, 987 BASE_ADDR - (i + 1) * page_size); 988 if (p == MAP_FAILED) 989 ksft_exit_fail_perror("mremap+alloc"); 990 } 991 992 ops->cleanup_area(BASE_ADDR, hpage_pmd_size); 993 ops->fault(p, 0, hpage_pmd_size); 994 if (!ops->check_huge(p, hpage_pmd_size, 1, hpage_pmd_size)) 995 success("OK"); 996 else 997 fail("Fail"); 998 999 c->collapse("Collapse PTE table full of different compound pages", p, 1, 1000 ops, true); 1001 1002 validate_memory(p, 0, hpage_pmd_size); 1003 ops->cleanup_area(p, hpage_pmd_size); 1004 ksft_test_result_report(exit_status, "%s\n", __func__); 1005 } 1006 1007 static void collapse_fork(struct collapse_context *c, struct mem_ops *ops) 1008 { 1009 int wstatus; 1010 void *p; 1011 1012 p = ops->setup_area(1); 1013 1014 ksft_print_msg("Allocate small page..."); 1015 ops->fault(p, 0, page_size); 1016 if (ops->check_huge(p, hpage_pmd_size, 0, hpage_pmd_size)) 1017 success("OK"); 1018 else 1019 fail("Fail"); 1020 1021 ksft_print_msg("Share small page over fork()..."); 1022 if (!fork()) { 1023 /* Do not touch settings on child exit */ 1024 if (ops->check_huge(p, hpage_pmd_size, 0, hpage_pmd_size)) 1025 success("OK"); 1026 else 1027 fail("Fail"); 1028 1029 ops->fault(p, page_size, 2 * page_size); 1030 c->collapse("Collapse PTE table with single page shared with parent process", 1031 p, 1, ops, true); 1032 1033 validate_memory(p, 0, page_size); 1034 ops->cleanup_area(p, hpage_pmd_size); 1035 _exit(exit_status); 1036 } 1037 1038 wait(&wstatus); 1039 exit_status = WEXITSTATUS(wstatus); 1040 if (exit_status == KSFT_FAIL) 1041 goto out; 1042 1043 ksft_print_msg("Check if parent still has small page..."); 1044 if (ops->check_huge(p, hpage_pmd_size, 0, hpage_pmd_size)) 1045 success("OK"); 1046 else 1047 fail("Fail"); 1048 validate_memory(p, 0, page_size); 1049 out: 1050 ops->cleanup_area(p, hpage_pmd_size); 1051 ksft_test_result_report(exit_status, "%s\n", __func__); 1052 } 1053 1054 static void collapse_fork_compound(struct collapse_context *c, struct mem_ops *ops) 1055 { 1056 int wstatus; 1057 void *p; 1058 1059 p = alloc_hpage(ops); 1060 ksft_print_msg("Share huge page over fork()..."); 1061 if (!fork()) { 1062 /* Do not touch settings on child exit */ 1063 if (ops->check_huge(p, hpage_pmd_size, 1, hpage_pmd_size)) 1064 success("OK"); 1065 else 1066 fail("Fail"); 1067 1068 ksft_print_msg("Split huge page PMD in child process..."); 1069 madvise(p, page_size, MADV_NOHUGEPAGE); 1070 madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE); 1071 if (ops->check_huge(p, hpage_pmd_size, 0, hpage_pmd_size)) 1072 success("OK"); 1073 else 1074 fail("Fail"); 1075 ops->fault(p, 0, page_size); 1076 1077 thp_write_num("khugepaged/max_ptes_shared", hpage_pmd_nr - 1); 1078 c->collapse("Collapse PTE table full of compound pages in child", 1079 p, 1, ops, true); 1080 thp_write_num("khugepaged/max_ptes_shared", 1081 thp_current_settings()->khugepaged.max_ptes_shared); 1082 1083 validate_memory(p, 0, hpage_pmd_size); 1084 ops->cleanup_area(p, hpage_pmd_size); 1085 _exit(exit_status); 1086 } 1087 1088 wait(&wstatus); 1089 exit_status = WEXITSTATUS(wstatus); 1090 if (exit_status == KSFT_FAIL) 1091 goto out; 1092 1093 ksft_print_msg("Check if parent still has huge page..."); 1094 if (ops->check_huge(p, hpage_pmd_size, 1, hpage_pmd_size)) 1095 success("OK"); 1096 else 1097 fail("Fail"); 1098 validate_memory(p, 0, hpage_pmd_size); 1099 out: 1100 ops->cleanup_area(p, hpage_pmd_size); 1101 ksft_test_result_report(exit_status, "%s\n", __func__); 1102 } 1103 1104 static void collapse_max_ptes_shared(struct collapse_context *c, struct mem_ops *ops) 1105 { 1106 int max_ptes_shared = thp_read_num("khugepaged/max_ptes_shared"); 1107 int wstatus; 1108 void *p; 1109 1110 p = alloc_hpage(ops); 1111 ksft_print_msg("Share huge page over fork()..."); 1112 if (!fork()) { 1113 /* Do not touch settings on child exit */ 1114 if (ops->check_huge(p, hpage_pmd_size, 1, hpage_pmd_size)) 1115 success("OK"); 1116 else 1117 fail("Fail"); 1118 1119 ksft_print_msg("Trigger CoW on page %d of %d...", 1120 hpage_pmd_nr - max_ptes_shared - 1, hpage_pmd_nr); 1121 ops->fault(p, 0, (hpage_pmd_nr - max_ptes_shared - 1) * page_size); 1122 if (ops->check_huge(p, hpage_pmd_size, 0, hpage_pmd_size)) 1123 success("OK"); 1124 else 1125 fail("Fail"); 1126 1127 c->collapse("Maybe collapse with max_ptes_shared exceeded", p, 1128 1, ops, !c->enforce_pte_scan_limits); 1129 1130 if (c->enforce_pte_scan_limits) { 1131 ksft_print_msg("Trigger CoW on page %d of %d...", 1132 hpage_pmd_nr - max_ptes_shared, hpage_pmd_nr); 1133 ops->fault(p, 0, (hpage_pmd_nr - max_ptes_shared) * 1134 page_size); 1135 if (ops->check_huge(p, hpage_pmd_size, 0, hpage_pmd_size)) 1136 success("OK"); 1137 else 1138 fail("Fail"); 1139 1140 c->collapse("Collapse with max_ptes_shared PTEs shared", 1141 p, 1, ops, true); 1142 } 1143 1144 validate_memory(p, 0, hpage_pmd_size); 1145 ops->cleanup_area(p, hpage_pmd_size); 1146 _exit(exit_status); 1147 } 1148 1149 wait(&wstatus); 1150 exit_status = WEXITSTATUS(wstatus); 1151 if (exit_status == KSFT_FAIL) 1152 goto out; 1153 1154 ksft_print_msg("Check if parent still has huge page..."); 1155 if (ops->check_huge(p, hpage_pmd_size, 1, hpage_pmd_size)) 1156 success("OK"); 1157 else 1158 fail("Fail"); 1159 validate_memory(p, 0, hpage_pmd_size); 1160 out: 1161 ops->cleanup_area(p, hpage_pmd_size); 1162 ksft_test_result_report(exit_status, "%s\n", __func__); 1163 } 1164 1165 /* 1166 * The parent writes to the fork-shared range throughout the child's 1167 * collapse. CoW must keep the two apart: the child sees the pre-fork 1168 * content, the parent only its own writes. 1169 */ 1170 static void collapse_fork_cow_race(struct collapse_context *c, struct mem_ops *ops) 1171 { 1172 const int stride = page_size / sizeof(int); 1173 int wstatus, child_status, i, n; 1174 unsigned long shared; 1175 volatile int *ip; 1176 pid_t child; 1177 int sync[2]; 1178 char go = 1; 1179 void *p; 1180 1181 /* At a page per 10 ms, 64 pages spread the writes across the collapse */ 1182 n = 64; 1183 shared = n * page_size; 1184 1185 p = ops->setup_area(1); 1186 /* Shared prefix, with the pre-fork pattern */ 1187 ops->fault(p, 0, shared); 1188 if (pipe(sync)) 1189 ksft_exit_fail_perror("pipe()"); 1190 1191 /* A volatile pointer so the stores are not merged or dropped */ 1192 ip = p; 1193 1194 ksft_print_msg("Fork, collapse in the child while the parent rewrites..."); 1195 child = fork(); 1196 if (!child) { 1197 int collapse_status; 1198 1199 close(sync[0]); 1200 /* Private remainder */ 1201 ops->fault(p, shared, hpage_pmd_size); 1202 /* Start the parent unsharing, and give it a head start */ 1203 if (write(sync[1], &go, 1) != 1) 1204 _exit(KSFT_FAIL); 1205 usleep(5000); 1206 c->collapse("Collapse a range the parent is writing to", 1207 p, 1, ops, true); 1208 collapse_status = exit_status; 1209 for (i = 0; i < n; i++) 1210 if (ip[i * stride] != i + 0xdead0000) 1211 break; 1212 if (i == n) 1213 success("OK"); 1214 else 1215 fail("Fail: child content"); 1216 /* The content check must not bury a failed collapse */ 1217 if (exit_status != KSFT_FAIL) 1218 exit_status = collapse_status; 1219 ops->cleanup_area(p, hpage_pmd_size); 1220 _exit(exit_status); 1221 } 1222 1223 close(sync[1]); 1224 if (read(sync[0], &go, 1) != 1) 1225 ksft_exit_fail_msg("child never reached the collapse\n"); 1226 close(sync[0]); 1227 1228 /* 1229 * Unshare one page at a time: a burst would break CoW on the whole 1230 * range before the collapse starts, leaving nothing shared to collapse. 1231 */ 1232 i = 0; 1233 for (;;) { 1234 pid_t ret; 1235 1236 if (i < n) 1237 ip[i * stride] = i + 0xbeef0000; 1238 i++; 1239 usleep(10 * 1000); 1240 ret = waitpid(child, &wstatus, WNOHANG); 1241 if (ret == child) 1242 break; 1243 if (ret < 0) 1244 ksft_exit_fail_perror("waitpid()"); 1245 } 1246 1247 /* Finish whatever the paced sweep did not reach */ 1248 for (; i < n; i++) 1249 ip[i * stride] = i + 0xbeef0000; 1250 /* A child that died reading the racing pages is a failure, not a zero */ 1251 child_status = WIFEXITED(wstatus) ? WEXITSTATUS(wstatus) : KSFT_FAIL; 1252 1253 ksft_print_msg("Check the parent sees only its own writes..."); 1254 for (i = 0; i < n; i++) 1255 if (ip[i * stride] != i + 0xbeef0000) 1256 break; 1257 if (i == n) 1258 success("OK"); 1259 else 1260 fail("Fail: parent content"); 1261 ops->cleanup_area(p, hpage_pmd_size); 1262 /* The parent's check must not bury the child's verdict */ 1263 if (exit_status != KSFT_FAIL) 1264 exit_status = child_status; 1265 ksft_test_result_report(exit_status, "%s\n", __func__); 1266 } 1267 1268 static void madvise_collapse_existing_thps(struct collapse_context *c, 1269 struct mem_ops *ops) 1270 { 1271 void *p; 1272 1273 p = ops->setup_area(1); 1274 ops->fault(p, 0, hpage_pmd_size); 1275 c->collapse("Collapse fully populated PTE table...", p, 1, ops, true); 1276 validate_memory(p, 0, hpage_pmd_size); 1277 1278 /* c->collapse() will find a hugepage and complain - call directly. */ 1279 __madvise_collapse("Re-collapse PMD-mapped hugepage", p, 1, ops, true); 1280 validate_memory(p, 0, hpage_pmd_size); 1281 ops->cleanup_area(p, hpage_pmd_size); 1282 ksft_test_result_report(exit_status, "%s\n", __func__); 1283 } 1284 1285 /* 1286 * Test race with khugepaged where page tables have been retracted and 1287 * pmd cleared. 1288 */ 1289 static void madvise_retracted_page_tables(struct collapse_context *c, 1290 struct mem_ops *ops) 1291 { 1292 void *p; 1293 int nr_hpages = 1; 1294 unsigned long size = nr_hpages * hpage_pmd_size; 1295 1296 p = ops->setup_area(nr_hpages); 1297 ops->fault(p, 0, size); 1298 1299 /* Let khugepaged collapse and leave pmd cleared */ 1300 if (wait_for_scan("Collapse and leave PMD cleared", p, size, nr_hpages, 1301 hpage_pmd_order, ops)) { 1302 fail("Timeout"); 1303 return; 1304 } 1305 success("OK"); 1306 c->collapse("Install huge PMD from page cache", p, nr_hpages, ops, 1307 true); 1308 validate_memory(p, 0, size); 1309 ops->cleanup_area(p, size); 1310 ksft_test_result_report(exit_status, "%s\n", __func__); 1311 } 1312 1313 /* Smallest order khugepaged will consider for mTHP collapse */ 1314 #define MIN_MTHP_ORDER 2 1315 1316 /* Time budget for one khugepaged pass in the collapse_order_* cases */ 1317 #define MTHP_PASS_TIMEOUT_S 30 1318 1319 static size_t mthp_window_size(void) 1320 { 1321 return page_size << collapse_order; 1322 } 1323 1324 static void mthp_push_target_order(void) 1325 { 1326 struct thp_settings settings = *thp_current_settings(); 1327 int i; 1328 1329 /* 1330 * Only the target order, and only for madvise: the cases fault their 1331 * region first, so the sources stay order 0 whatever -s asked for. 1332 */ 1333 settings.thp_enabled = THP_NEVER; 1334 for (i = 0; i < NR_ORDERS; i++) 1335 settings.hugepages[i].enabled = THP_NEVER; 1336 settings.hugepages[collapse_order].enabled = THP_MADVISE; 1337 thp_push_settings(&settings); 1338 } 1339 1340 static bool all_windows_at_order(void *p, size_t len) 1341 { 1342 return is_range_backed_by_order(p, len, collapse_order, 1343 pagemap_fd, kpageflags_fd); 1344 } 1345 1346 static bool any_window_at_order(void *p, size_t len) 1347 { 1348 size_t window = mthp_window_size(); 1349 char *addr = p; 1350 1351 for (; len >= window; addr += window, len -= window) { 1352 if (all_windows_at_order(addr, window)) 1353 return true; 1354 } 1355 return false; 1356 } 1357 1358 static void collapse_order_single_window(struct collapse_context *c, 1359 struct mem_ops *ops) 1360 { 1361 size_t window = mthp_window_size(); 1362 void *p; 1363 1364 mthp_push_target_order(); 1365 1366 p = ops->setup_area(1); 1367 ops->fault(p, window, 2 * window); 1368 if (any_window_at_order(p, hpage_pmd_size)) 1369 ksft_exit_fail_msg("Unexpected large folio after fault\n"); 1370 1371 if (madvise(p, hpage_pmd_size, MADV_HUGEPAGE)) 1372 ksft_exit_fail_perror("madvise(MADV_HUGEPAGE)"); 1373 ksft_print_msg("Collapse one fully populated window..."); 1374 if (!khugepaged_full_pass(MTHP_PASS_TIMEOUT_S)) 1375 fail("Timeout"); 1376 else if (all_windows_at_order(p + window, window) && 1377 !any_window_at_order(p, window) && 1378 !any_window_at_order(p + 2 * window, 1379 hpage_pmd_size - 2 * window)) 1380 success("OK"); 1381 else 1382 fail("Fail"); 1383 1384 validate_memory(p, window, 2 * window); 1385 ops->cleanup_area(p, hpage_pmd_size); 1386 thp_pop_settings(); 1387 ksft_test_result_report(exit_status, "%s\n", __func__); 1388 } 1389 1390 static void collapse_order_partial_window(struct collapse_context *c, 1391 struct mem_ops *ops) 1392 { 1393 void *p; 1394 1395 mthp_push_target_order(); 1396 1397 p = ops->setup_area(1); 1398 ops->fault(p, 0, page_size); 1399 if (any_window_at_order(p, hpage_pmd_size)) 1400 ksft_exit_fail_msg("Unexpected large folio after fault\n"); 1401 1402 if (madvise(p, hpage_pmd_size, MADV_HUGEPAGE)) 1403 ksft_exit_fail_perror("madvise(MADV_HUGEPAGE)"); 1404 ksft_print_msg("Collapse window with single PTE entry present..."); 1405 if (!khugepaged_full_pass(MTHP_PASS_TIMEOUT_S)) 1406 fail("Timeout"); 1407 else if (all_windows_at_order(p, mthp_window_size())) 1408 success("OK"); 1409 else 1410 fail("Fail"); 1411 1412 validate_memory(p, 0, page_size); 1413 ops->cleanup_area(p, hpage_pmd_size); 1414 thp_pop_settings(); 1415 ksft_test_result_report(exit_status, "%s\n", __func__); 1416 } 1417 1418 static void collapse_order_max_ptes_none(struct collapse_context *c, 1419 struct mem_ops *ops) 1420 { 1421 struct thp_settings settings; 1422 size_t window = mthp_window_size(); 1423 void *p; 1424 1425 mthp_push_target_order(); 1426 settings = *thp_current_settings(); 1427 settings.khugepaged.max_ptes_none = 0; 1428 thp_push_settings(&settings); 1429 1430 p = ops->setup_area(1); 1431 ops->fault(p, 0, 2 * window - page_size); 1432 if (any_window_at_order(p, hpage_pmd_size)) 1433 ksft_exit_fail_msg("Unexpected large folio after fault\n"); 1434 1435 if (madvise(p, hpage_pmd_size, MADV_HUGEPAGE)) 1436 ksft_exit_fail_perror("madvise(MADV_HUGEPAGE)"); 1437 ksft_print_msg("Collapse full window, not the one missing a page..."); 1438 if (!khugepaged_full_pass(MTHP_PASS_TIMEOUT_S)) 1439 fail("Timeout"); 1440 else if (all_windows_at_order(p, window) && 1441 !any_window_at_order(p + window, window)) 1442 success("OK"); 1443 else 1444 fail("Fail"); 1445 1446 validate_memory(p, 0, 2 * window - page_size); 1447 ops->cleanup_area(p, hpage_pmd_size); 1448 thp_pop_settings(); 1449 thp_pop_settings(); 1450 ksft_test_result_report(exit_status, "%s\n", __func__); 1451 } 1452 1453 static void collapse_order_mixed_sources(struct collapse_context *c, 1454 struct mem_ops *ops) 1455 { 1456 int source_order = anon_order ? anon_order : MIN_MTHP_ORDER; 1457 struct thp_settings settings; 1458 void *p; 1459 1460 if (source_order >= collapse_order || 1461 !(thp_supported_orders() & (1UL << source_order))) { 1462 ksft_test_result_skip("%s: no supported source order below target\n", 1463 __func__); 1464 return; 1465 } 1466 1467 mthp_push_target_order(); 1468 1469 settings = *thp_current_settings(); 1470 settings.hugepages[source_order].enabled = THP_ALWAYS; 1471 thp_push_settings(&settings); 1472 p = ops->setup_area(1); 1473 ops->fault(p, 0, hpage_pmd_size); 1474 thp_pop_settings(); 1475 1476 /* 1477 * The allocator can fall back to smaller folios under fragmentation; 1478 * having nothing to collapse from is not a failure. 1479 */ 1480 if (!is_range_backed_by_order(p, hpage_pmd_size, source_order, 1481 pagemap_fd, kpageflags_fd)) { 1482 ksft_print_msg("No order-%d sources to collapse...", source_order); 1483 skip("Skip"); 1484 ops->cleanup_area(p, hpage_pmd_size); 1485 thp_pop_settings(); 1486 ksft_test_result_report(exit_status, "%s\n", __func__); 1487 return; 1488 } 1489 1490 if (madvise(p, hpage_pmd_size, MADV_HUGEPAGE)) 1491 ksft_exit_fail_perror("madvise(MADV_HUGEPAGE)"); 1492 ksft_print_msg("Collapse region backed by order-%d sources...", 1493 source_order); 1494 if (!khugepaged_full_pass(MTHP_PASS_TIMEOUT_S)) 1495 fail("Timeout"); 1496 else if (all_windows_at_order(p, hpage_pmd_size)) 1497 success("OK"); 1498 else 1499 fail("Fail"); 1500 1501 validate_memory(p, 0, hpage_pmd_size); 1502 ops->cleanup_area(p, hpage_pmd_size); 1503 thp_pop_settings(); 1504 ksft_test_result_report(exit_status, "%s\n", __func__); 1505 } 1506 1507 static void usage(void) 1508 { 1509 fprintf(stderr, "\nUsage: ./khugepaged [OPTIONS] <test type> [dir]\n\n"); 1510 fprintf(stderr, "\t<test type>\t: <context>:<mem_type>\n"); 1511 fprintf(stderr, "\t<context>\t: [all|khugepaged|mthp_khugepaged|madvise]\n"); 1512 fprintf(stderr, "\t<mem_type>\t: [all|anon|file|shmem]\n"); 1513 fprintf(stderr, "\n\t\"file,all\" mem_type requires [dir] argument\n"); 1514 fprintf(stderr, "\n\t\"file,all\" mem_type requires a file system\n"); 1515 fprintf(stderr, "\twith PMD-sized large folio support\n"); 1516 fprintf(stderr, "\n\tif [dir] is a (sub)directory of a tmpfs mount, tmpfs must be\n"); 1517 fprintf(stderr, "\tmounted with huge=advise option for khugepaged tests to work\n"); 1518 fprintf(stderr, "\n\tmthp_khugepaged only supports anon mem_type now.\n"); 1519 fprintf(stderr, "\n\tSupported Options:\n"); 1520 fprintf(stderr, "\t\t-h: This help message.\n"); 1521 fprintf(stderr, "\t\t-s: mTHP size, expressed as page order.\n"); 1522 fprintf(stderr, "\t\t Defaults to 0. Use this size for anon or shmem allocations.\n"); 1523 fprintf(stderr, "\t\t-c: collapse order for mTHP collapse, expressed as page order.\n"); 1524 fprintf(stderr, "\t\t Defaults to every supported order below the PMD.\n"); 1525 fprintf(stderr, "\t\t -s, if set, is the source order for the mixed-source case.\n"); 1526 exit(1); 1527 } 1528 1529 static void parse_test_type(int argc, char **argv) 1530 { 1531 bool mthp_context_implied = false; 1532 int opt; 1533 char *buf; 1534 const char *token; 1535 1536 while ((opt = getopt(argc, argv, "s:c:h")) != -1) { 1537 switch (opt) { 1538 case 's': 1539 anon_order = atoi(optarg); 1540 break; 1541 case 'c': 1542 collapse_order = atoi(optarg); 1543 collapse_order_set = true; 1544 break; 1545 case 'h': 1546 default: 1547 usage(); 1548 } 1549 } 1550 1551 /* 1552 * Both orders end up as array indices and shift counts, so neither 1553 * can be negative, and a zero collapse order asks for base pages. 1554 */ 1555 if (anon_order < 0 || anon_order > hpage_pmd_order) 1556 ksft_exit_fail_msg("-s takes an order in 0..%d, not %d\n", 1557 hpage_pmd_order, anon_order); 1558 if (collapse_order_set && 1559 (collapse_order <= 0 || collapse_order >= hpage_pmd_order)) 1560 ksft_exit_fail_msg("-c takes an order in 1..%d, not %d\n", 1561 hpage_pmd_order - 1, collapse_order); 1562 1563 argv += optind; 1564 argc -= optind; 1565 1566 if (argc == 0) { 1567 /* No arguments: anon under every context */ 1568 khugepaged_context = &__khugepaged_context; 1569 mthp_khugepaged_context = &__mthp_khugepaged_context; 1570 madvise_context = &__madvise_context; 1571 anon_ops = &__anon_ops; 1572 return; 1573 } 1574 1575 buf = argv[0]; 1576 token = strsep(&buf, ":"); 1577 1578 if (!strcmp(token, "all")) { 1579 khugepaged_context = &__khugepaged_context; 1580 mthp_khugepaged_context = &__mthp_khugepaged_context; 1581 madvise_context = &__madvise_context; 1582 /* The mTHP context has only anon cases; let other mem_types drop it */ 1583 mthp_context_implied = true; 1584 } else if (!strcmp(token, "khugepaged")) { 1585 khugepaged_context = &__khugepaged_context; 1586 } else if (!strcmp(token, "mthp_khugepaged")) { 1587 mthp_khugepaged_context = &__mthp_khugepaged_context; 1588 } else if (!strcmp(token, "madvise")) { 1589 madvise_context = &__madvise_context; 1590 } else { 1591 usage(); 1592 } 1593 1594 if (!buf) 1595 usage(); 1596 1597 if (!strcmp(buf, "all")) { 1598 read_only_file_ops = &__read_only_file_ops; 1599 read_write_file_read_ops = &__read_write_file_read_ops; 1600 read_write_file_write_ops = &__read_write_file_write_ops; 1601 anon_ops = &__anon_ops; 1602 shmem_ops = &__shmem_ops; 1603 } else if (!strcmp(buf, "anon")) { 1604 anon_ops = &__anon_ops; 1605 } else if (!strcmp(buf, "file")) { 1606 read_only_file_ops = &__read_only_file_ops; 1607 read_write_file_read_ops = &__read_write_file_read_ops; 1608 read_write_file_write_ops = &__read_write_file_write_ops; 1609 if (mthp_khugepaged_context && !mthp_context_implied) 1610 usage(); 1611 mthp_khugepaged_context = NULL; 1612 } else if (!strcmp(buf, "shmem")) { 1613 shmem_ops = &__shmem_ops; 1614 if (mthp_khugepaged_context && !mthp_context_implied) 1615 usage(); 1616 mthp_khugepaged_context = NULL; 1617 } else { 1618 usage(); 1619 } 1620 1621 if (!read_only_file_ops && !read_write_file_read_ops && 1622 !read_write_file_write_ops) 1623 return; 1624 1625 if (argc != 2) 1626 usage(); 1627 1628 get_finfo(argv[1]); 1629 } 1630 1631 typedef void (*test_fn)(struct collapse_context *c, struct mem_ops *ops); 1632 1633 struct test_case { 1634 struct collapse_context *ctx; 1635 struct mem_ops *ops; 1636 const char *desc; 1637 test_fn fn; 1638 int order; /* mTHP contexts: the collapse order */ 1639 }; 1640 1641 #define MAX_TEST_CASES 256 1642 static struct test_case test_cases[MAX_TEST_CASES]; 1643 static int nr_test_cases; 1644 1645 #define TEST(t, c, o) do { \ 1646 if (c && o) { \ 1647 if (nr_test_cases >= MAX_TEST_CASES) \ 1648 ksft_exit_fail_msg("MAX_TEST_CASES is too small\n"); \ 1649 test_cases[nr_test_cases++] = (struct test_case){ \ 1650 .ctx = c, \ 1651 .ops = o, \ 1652 .desc = #t, \ 1653 .fn = t, \ 1654 .order = collapse_order, \ 1655 }; \ 1656 } \ 1657 } while (0) 1658 1659 int main(int argc, char **argv) 1660 { 1661 struct thp_settings default_settings = { 1662 .thp_enabled = THP_MADVISE, 1663 .thp_defrag = THP_DEFRAG_ALWAYS, 1664 .shmem_enabled = SHMEM_ADVISE, 1665 .use_zero_page = 0, 1666 .khugepaged = { 1667 .defrag = 1, 1668 .alloc_sleep_millisecs = 10, 1669 .scan_sleep_millisecs = 10, 1670 }, 1671 /* 1672 * When testing file-backed memory, the collapse path 1673 * looks at how many pages are found in the page cache, not 1674 * what pages are mapped. Disable read ahead optimization so 1675 * pages don't find their way into the page cache unless 1676 * we mem_ops->fault() them in. 1677 */ 1678 .read_ahead_kb = 0, 1679 }; 1680 1681 ksft_print_header(); 1682 1683 if (!thp_is_enabled()) 1684 ksft_exit_skip("Transparent Hugepages not available\n"); 1685 1686 page_size = getpagesize(); 1687 hpage_pmd_size = read_pmd_pagesize(); 1688 if (!hpage_pmd_size) 1689 ksft_exit_fail_msg("Reading PMD pagesize failed\n"); 1690 hpage_pmd_nr = hpage_pmd_size / page_size; 1691 hpage_pmd_order = __builtin_ctz(hpage_pmd_nr); 1692 1693 parse_test_type(argc, argv); 1694 1695 if (mthp_khugepaged_context) { 1696 unsigned long orders = thp_supported_orders(); 1697 1698 if (collapse_order_set) { 1699 if (!(orders & (1UL << collapse_order))) 1700 ksft_exit_skip("Order %d is not a supported anon THP order\n", 1701 collapse_order); 1702 if (collapse_order <= anon_order) 1703 ksft_exit_skip("-c %d needs a source order below it, -s says %d\n", 1704 collapse_order, anon_order); 1705 collapse_orders[nr_collapse_orders++] = collapse_order; 1706 } else { 1707 /* 1708 * Every supported order above the source: -s makes the 1709 * fault path hand out folios of that order, so a target 1710 * at or below it has nothing to collapse. 1711 */ 1712 int first = anon_order + 1; 1713 1714 if (first < MIN_MTHP_ORDER) 1715 first = MIN_MTHP_ORDER; 1716 for (int i = first; i < hpage_pmd_order; i++) { 1717 if (orders & (1UL << i)) 1718 collapse_orders[nr_collapse_orders++] = i; 1719 } 1720 if (!nr_collapse_orders) 1721 ksft_print_msg("mTHP cases skipped: no order above the source\n"); 1722 } 1723 } 1724 1725 if (mthp_khugepaged_context) { 1726 pagemap_fd = open("/proc/self/pagemap", O_RDONLY); 1727 if (pagemap_fd < 0) 1728 ksft_exit_fail_perror("open(/proc/self/pagemap)"); 1729 kpageflags_fd = open("/proc/kpageflags", O_RDONLY); 1730 if (kpageflags_fd < 0) 1731 ksft_exit_fail_perror("open(/proc/kpageflags)"); 1732 } 1733 1734 setbuf(stdout, NULL); 1735 1736 /* 1737 * Without a PMD-order page cache folio the kernel refuses these 1738 * collapses, so there is nothing to test. 1739 */ 1740 if (!(thp_file_supported_orders() & (1UL << hpage_pmd_order))) { 1741 if (shmem_ops) { 1742 ksft_print_msg("no PMD-order page cache folio: skipping shmem\n"); 1743 shmem_ops = NULL; 1744 } 1745 if (read_only_file_ops) { 1746 ksft_print_msg("no PMD-order page cache folio: skipping file\n"); 1747 read_only_file_ops = NULL; 1748 read_write_file_read_ops = NULL; 1749 read_write_file_write_ops = NULL; 1750 } 1751 if (!anon_ops && !shmem_ops && !read_only_file_ops) 1752 ksft_exit_skip("No mem_type left to run\n"); 1753 } 1754 1755 default_settings.khugepaged.max_ptes_none = hpage_pmd_nr - 1; 1756 default_settings.khugepaged.max_ptes_swap = hpage_pmd_nr / 8; 1757 default_settings.khugepaged.max_ptes_shared = hpage_pmd_nr / 2; 1758 default_settings.khugepaged.pages_to_scan = hpage_pmd_nr * 8; 1759 default_settings.hugepages[hpage_pmd_order].enabled = THP_INHERIT; 1760 default_settings.hugepages[anon_order].enabled = THP_ALWAYS; 1761 default_settings.shmem_hugepages[hpage_pmd_order].enabled = SHMEM_INHERIT; 1762 default_settings.shmem_hugepages[anon_order].enabled = SHMEM_ALWAYS; 1763 1764 save_settings(); 1765 thp_push_settings(&default_settings); 1766 1767 TEST(collapse_full, khugepaged_context, anon_ops); 1768 TEST(collapse_full, khugepaged_context, read_only_file_ops); 1769 TEST(collapse_full, khugepaged_context, read_write_file_read_ops); 1770 TEST(collapse_full, khugepaged_context, read_write_file_write_ops); 1771 TEST(collapse_full, khugepaged_context, shmem_ops); 1772 for (int i = 0; i < nr_collapse_orders; i++) { 1773 collapse_order = collapse_orders[i]; 1774 TEST(collapse_full, mthp_khugepaged_context, anon_ops); 1775 TEST(collapse_empty, mthp_khugepaged_context, anon_ops); 1776 TEST(collapse_single_mthp, mthp_khugepaged_context, anon_ops); 1777 TEST(collapse_order_single_window, mthp_khugepaged_context, anon_ops); 1778 TEST(collapse_order_partial_window, mthp_khugepaged_context, anon_ops); 1779 TEST(collapse_order_max_ptes_none, mthp_khugepaged_context, anon_ops); 1780 TEST(collapse_order_mixed_sources, mthp_khugepaged_context, anon_ops); 1781 } 1782 1783 TEST(collapse_full, madvise_context, anon_ops); 1784 TEST(collapse_full, madvise_context, read_only_file_ops); 1785 TEST(collapse_full, madvise_context, read_write_file_read_ops); 1786 TEST(collapse_full, madvise_context, read_write_file_write_ops); 1787 TEST(collapse_full, madvise_context, shmem_ops); 1788 1789 TEST(collapse_empty, khugepaged_context, anon_ops); 1790 TEST(collapse_empty, madvise_context, anon_ops); 1791 1792 TEST(collapse_single_pte_entry, khugepaged_context, anon_ops); 1793 TEST(collapse_single_pte_entry, khugepaged_context, read_only_file_ops); 1794 TEST(collapse_single_pte_entry, khugepaged_context, read_write_file_read_ops); 1795 TEST(collapse_single_pte_entry, khugepaged_context, read_write_file_write_ops); 1796 TEST(collapse_single_pte_entry, khugepaged_context, shmem_ops); 1797 TEST(collapse_single_pte_entry, madvise_context, anon_ops); 1798 TEST(collapse_single_pte_entry, madvise_context, read_only_file_ops); 1799 TEST(collapse_single_pte_entry, madvise_context, read_write_file_read_ops); 1800 TEST(collapse_single_pte_entry, madvise_context, read_write_file_write_ops); 1801 TEST(collapse_single_pte_entry, madvise_context, shmem_ops); 1802 1803 TEST(collapse_max_ptes_none, khugepaged_context, anon_ops); 1804 TEST(collapse_max_ptes_none, khugepaged_context, read_only_file_ops); 1805 TEST(collapse_max_ptes_none, khugepaged_context, read_write_file_read_ops); 1806 TEST(collapse_max_ptes_none, khugepaged_context, read_write_file_write_ops); 1807 TEST(collapse_max_ptes_none, madvise_context, anon_ops); 1808 TEST(collapse_max_ptes_none, madvise_context, read_only_file_ops); 1809 TEST(collapse_max_ptes_none, madvise_context, read_write_file_read_ops); 1810 TEST(collapse_max_ptes_none, madvise_context, read_write_file_write_ops); 1811 1812 TEST(collapse_single_pte_entry_compound, khugepaged_context, anon_ops); 1813 TEST(collapse_single_pte_entry_compound, khugepaged_context, read_only_file_ops); 1814 TEST(collapse_single_pte_entry_compound, khugepaged_context, read_write_file_read_ops); 1815 TEST(collapse_single_pte_entry_compound, madvise_context, anon_ops); 1816 TEST(collapse_single_pte_entry_compound, madvise_context, read_only_file_ops); 1817 TEST(collapse_single_pte_entry_compound, madvise_context, read_write_file_read_ops); 1818 1819 TEST(collapse_full_of_compound, khugepaged_context, anon_ops); 1820 TEST(collapse_full_of_compound, khugepaged_context, read_only_file_ops); 1821 TEST(collapse_full_of_compound, khugepaged_context, read_write_file_read_ops); 1822 TEST(collapse_full_of_compound, khugepaged_context, shmem_ops); 1823 TEST(collapse_full_of_compound, madvise_context, anon_ops); 1824 TEST(collapse_full_of_compound, madvise_context, read_only_file_ops); 1825 TEST(collapse_full_of_compound, madvise_context, read_write_file_read_ops); 1826 TEST(collapse_full_of_compound, madvise_context, shmem_ops); 1827 1828 TEST(collapse_compound_extreme, khugepaged_context, anon_ops); 1829 TEST(collapse_compound_extreme, madvise_context, anon_ops); 1830 1831 TEST(collapse_swapin_single_pte, khugepaged_context, anon_ops); 1832 TEST(collapse_swapin_single_pte, madvise_context, anon_ops); 1833 1834 TEST(collapse_max_ptes_swap, khugepaged_context, anon_ops); 1835 TEST(collapse_max_ptes_swap, madvise_context, anon_ops); 1836 1837 TEST(collapse_fork, khugepaged_context, anon_ops); 1838 TEST(collapse_fork, madvise_context, anon_ops); 1839 1840 TEST(collapse_fork_compound, khugepaged_context, anon_ops); 1841 TEST(collapse_fork_compound, madvise_context, anon_ops); 1842 1843 TEST(collapse_max_ptes_shared, khugepaged_context, anon_ops); 1844 TEST(collapse_max_ptes_shared, madvise_context, anon_ops); 1845 1846 TEST(collapse_fork_cow_race, khugepaged_context, anon_ops); 1847 TEST(collapse_fork_cow_race, madvise_context, anon_ops); 1848 1849 TEST(madvise_collapse_existing_thps, madvise_context, anon_ops); 1850 TEST(madvise_collapse_existing_thps, madvise_context, read_only_file_ops); 1851 TEST(madvise_collapse_existing_thps, madvise_context, read_write_file_read_ops); 1852 TEST(madvise_collapse_existing_thps, madvise_context, shmem_ops); 1853 1854 TEST(madvise_retracted_page_tables, madvise_context, read_only_file_ops); 1855 TEST(madvise_retracted_page_tables, madvise_context, read_write_file_read_ops); 1856 TEST(madvise_retracted_page_tables, madvise_context, shmem_ops); 1857 1858 ksft_set_plan(nr_test_cases + 1); 1859 1860 alloc_at_fault(); 1861 for (int i = 0; i < nr_test_cases; i++) { 1862 struct test_case *t = &test_cases[i]; 1863 1864 if (t->ctx == &__mthp_khugepaged_context) { 1865 collapse_order = t->order; 1866 ksft_print_msg("\n# Run test: %s (%s:%s, order %d)\n", 1867 t->desc, t->ctx->name, t->ops->name, 1868 t->order); 1869 } else { 1870 ksft_print_msg("\n# Run test: %s (%s:%s)\n", t->desc, 1871 t->ctx->name, t->ops->name); 1872 } 1873 t->fn(t->ctx, t->ops); 1874 } 1875 1876 ksft_finished(); 1877 } 1878