1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * A test of splitting PMD THPs and PTE-mapped THPs from a specified virtual 4 * address range in a process via <debugfs>/split_huge_pages interface. 5 */ 6 7 #define _GNU_SOURCE 8 #include <assert.h> 9 #include <stdio.h> 10 #include <stdlib.h> 11 #include <stdarg.h> 12 #include <unistd.h> 13 #include <inttypes.h> 14 #include <string.h> 15 #include <fcntl.h> 16 #include <sys/mman.h> 17 #include <sys/mount.h> 18 #include <sys/param.h> 19 #include <malloc.h> 20 #include <stdbool.h> 21 #include <time.h> 22 #include "vm_util.h" 23 #include "../kselftest.h" 24 25 uint64_t pagesize; 26 unsigned int pageshift; 27 uint64_t pmd_pagesize; 28 unsigned int pmd_order; 29 int *expected_orders; 30 31 #define SPLIT_DEBUGFS "/sys/kernel/debug/split_huge_pages" 32 #define SMAP_PATH "/proc/self/smaps" 33 #define INPUT_MAX 80 34 35 #define PID_FMT "%d,0x%lx,0x%lx,%d" 36 #define PID_FMT_OFFSET "%d,0x%lx,0x%lx,%d,%d" 37 #define PATH_FMT "%s,0x%lx,0x%lx,%d" 38 39 const char *pagemap_proc = "/proc/self/pagemap"; 40 const char *kpageflags_proc = "/proc/kpageflags"; 41 int pagemap_fd; 42 int kpageflags_fd; 43 44 static bool is_backed_by_folio(char *vaddr, int order, int pagemap_fd, 45 int kpageflags_fd) 46 { 47 const unsigned long nr_pages = 1UL << order; 48 unsigned long pfn_head; 49 uint64_t pfn_flags; 50 unsigned long pfn; 51 unsigned long i; 52 53 pfn = pagemap_get_pfn(pagemap_fd, vaddr); 54 55 /* non present page */ 56 if (pfn == -1UL) 57 return false; 58 59 if (pageflags_get(pfn, kpageflags_fd, &pfn_flags)) 60 goto fail; 61 62 /* check for order-0 pages */ 63 if (!order) { 64 if (pfn_flags & (KPF_THP | KPF_COMPOUND_HEAD | KPF_COMPOUND_TAIL)) 65 return false; 66 return true; 67 } 68 69 /* non THP folio */ 70 if (!(pfn_flags & KPF_THP)) 71 return false; 72 73 pfn_head = pfn & ~(nr_pages - 1); 74 75 if (pageflags_get(pfn_head, kpageflags_fd, &pfn_flags)) 76 goto fail; 77 78 /* head PFN has no compound_head flag set */ 79 if (!(pfn_flags & (KPF_THP | KPF_COMPOUND_HEAD))) 80 return false; 81 82 /* check all tail PFN flags */ 83 for (i = 1; i < nr_pages; i++) { 84 if (pageflags_get(pfn_head + i, kpageflags_fd, &pfn_flags)) 85 goto fail; 86 if (!(pfn_flags & (KPF_THP | KPF_COMPOUND_TAIL))) 87 return false; 88 } 89 90 /* 91 * check the PFN after this folio, but if its flags cannot be obtained, 92 * assume this folio has the expected order 93 */ 94 if (pageflags_get(pfn_head + nr_pages, kpageflags_fd, &pfn_flags)) 95 return true; 96 97 /* this folio is bigger than the given order */ 98 if (pfn_flags & (KPF_THP | KPF_COMPOUND_TAIL)) 99 return false; 100 101 return true; 102 fail: 103 ksft_exit_fail_msg("Failed to get folio info\n"); 104 return false; 105 } 106 107 static int vaddr_pageflags_get(char *vaddr, int pagemap_fd, int kpageflags_fd, 108 uint64_t *flags) 109 { 110 unsigned long pfn; 111 112 pfn = pagemap_get_pfn(pagemap_fd, vaddr); 113 114 /* non-present PFN */ 115 if (pfn == -1UL) 116 return 1; 117 118 if (pageflags_get(pfn, kpageflags_fd, flags)) 119 return -1; 120 121 return 0; 122 } 123 124 /* 125 * gather_after_split_folio_orders - scan through [vaddr_start, len) and record 126 * folio orders 127 * 128 * @vaddr_start: start vaddr 129 * @len: range length 130 * @pagemap_fd: file descriptor to /proc/<pid>/pagemap 131 * @kpageflags_fd: file descriptor to /proc/kpageflags 132 * @orders: output folio order array 133 * @nr_orders: folio order array size 134 * 135 * gather_after_split_folio_orders() scan through [vaddr_start, len) and check 136 * all folios within the range and record their orders. All order-0 pages will 137 * be recorded. Non-present vaddr is skipped. 138 * 139 * NOTE: the function is used to check folio orders after a split is performed, 140 * so it assumes [vaddr_start, len) fully maps to after-split folios within that 141 * range. 142 * 143 * Return: 0 - no error, -1 - unhandled cases 144 */ 145 static int gather_after_split_folio_orders(char *vaddr_start, size_t len, 146 int pagemap_fd, int kpageflags_fd, int orders[], int nr_orders) 147 { 148 uint64_t page_flags = 0; 149 int cur_order = -1; 150 char *vaddr; 151 152 if (pagemap_fd == -1 || kpageflags_fd == -1) 153 return -1; 154 if (!orders) 155 return -1; 156 if (nr_orders <= 0) 157 return -1; 158 159 for (vaddr = vaddr_start; vaddr < vaddr_start + len;) { 160 char *next_folio_vaddr; 161 int status; 162 163 status = vaddr_pageflags_get(vaddr, pagemap_fd, kpageflags_fd, 164 &page_flags); 165 if (status < 0) 166 return -1; 167 168 /* skip non present vaddr */ 169 if (status == 1) { 170 vaddr += psize(); 171 continue; 172 } 173 174 /* all order-0 pages with possible false postive (non folio) */ 175 if (!(page_flags & (KPF_COMPOUND_HEAD | KPF_COMPOUND_TAIL))) { 176 orders[0]++; 177 vaddr += psize(); 178 continue; 179 } 180 181 /* skip non thp compound pages */ 182 if (!(page_flags & KPF_THP)) { 183 vaddr += psize(); 184 continue; 185 } 186 187 /* vpn points to part of a THP at this point */ 188 if (page_flags & KPF_COMPOUND_HEAD) 189 cur_order = 1; 190 else { 191 vaddr += psize(); 192 continue; 193 } 194 195 next_folio_vaddr = vaddr + (1UL << (cur_order + pshift())); 196 197 if (next_folio_vaddr >= vaddr_start + len) 198 break; 199 200 while ((status = vaddr_pageflags_get(next_folio_vaddr, 201 pagemap_fd, kpageflags_fd, 202 &page_flags)) >= 0) { 203 /* 204 * non present vaddr, next compound head page, or 205 * order-0 page 206 */ 207 if (status == 1 || 208 (page_flags & KPF_COMPOUND_HEAD) || 209 !(page_flags & (KPF_COMPOUND_HEAD | KPF_COMPOUND_TAIL))) { 210 if (cur_order < nr_orders) { 211 orders[cur_order]++; 212 cur_order = -1; 213 vaddr = next_folio_vaddr; 214 } 215 break; 216 } 217 218 cur_order++; 219 next_folio_vaddr = vaddr + (1UL << (cur_order + pshift())); 220 } 221 222 if (status < 0) 223 return status; 224 } 225 if (cur_order > 0 && cur_order < nr_orders) 226 orders[cur_order]++; 227 return 0; 228 } 229 230 static int check_after_split_folio_orders(char *vaddr_start, size_t len, 231 int pagemap_fd, int kpageflags_fd, int orders[], int nr_orders) 232 { 233 int *vaddr_orders; 234 int status; 235 int i; 236 237 vaddr_orders = (int *)malloc(sizeof(int) * nr_orders); 238 239 if (!vaddr_orders) 240 ksft_exit_fail_msg("Cannot allocate memory for vaddr_orders"); 241 242 memset(vaddr_orders, 0, sizeof(int) * nr_orders); 243 status = gather_after_split_folio_orders(vaddr_start, len, pagemap_fd, 244 kpageflags_fd, vaddr_orders, nr_orders); 245 if (status) 246 ksft_exit_fail_msg("gather folio info failed\n"); 247 248 for (i = 0; i < nr_orders; i++) 249 if (vaddr_orders[i] != orders[i]) { 250 ksft_print_msg("order %d: expected: %d got %d\n", i, 251 orders[i], vaddr_orders[i]); 252 status = -1; 253 } 254 255 free(vaddr_orders); 256 return status; 257 } 258 259 static void write_file(const char *path, const char *buf, size_t buflen) 260 { 261 int fd; 262 ssize_t numwritten; 263 264 fd = open(path, O_WRONLY); 265 if (fd == -1) 266 ksft_exit_fail_msg("%s open failed: %s\n", path, strerror(errno)); 267 268 numwritten = write(fd, buf, buflen - 1); 269 close(fd); 270 if (numwritten < 1) 271 ksft_exit_fail_msg("Write failed\n"); 272 } 273 274 static void write_debugfs(const char *fmt, ...) 275 { 276 char input[INPUT_MAX]; 277 int ret; 278 va_list argp; 279 280 va_start(argp, fmt); 281 ret = vsnprintf(input, INPUT_MAX, fmt, argp); 282 va_end(argp); 283 284 if (ret >= INPUT_MAX) 285 ksft_exit_fail_msg("%s: Debugfs input is too long\n", __func__); 286 287 write_file(SPLIT_DEBUGFS, input, ret + 1); 288 } 289 290 static char *allocate_zero_filled_hugepage(size_t len) 291 { 292 char *result; 293 size_t i; 294 295 result = memalign(pmd_pagesize, len); 296 if (!result) { 297 printf("Fail to allocate memory\n"); 298 exit(EXIT_FAILURE); 299 } 300 301 madvise(result, len, MADV_HUGEPAGE); 302 303 for (i = 0; i < len; i++) 304 result[i] = (char)0; 305 306 return result; 307 } 308 309 static void verify_rss_anon_split_huge_page_all_zeroes(char *one_page, int nr_hpages, size_t len) 310 { 311 unsigned long rss_anon_before, rss_anon_after; 312 size_t i; 313 314 if (!check_huge_anon(one_page, nr_hpages, pmd_pagesize)) 315 ksft_exit_fail_msg("No THP is allocated\n"); 316 317 rss_anon_before = rss_anon(); 318 if (!rss_anon_before) 319 ksft_exit_fail_msg("No RssAnon is allocated before split\n"); 320 321 /* split all THPs */ 322 write_debugfs(PID_FMT, getpid(), (uint64_t)one_page, 323 (uint64_t)one_page + len, 0); 324 325 for (i = 0; i < len; i++) 326 if (one_page[i] != (char)0) 327 ksft_exit_fail_msg("%ld byte corrupted\n", i); 328 329 if (!check_huge_anon(one_page, 0, pmd_pagesize)) 330 ksft_exit_fail_msg("Still AnonHugePages not split\n"); 331 332 rss_anon_after = rss_anon(); 333 if (rss_anon_after >= rss_anon_before) 334 ksft_exit_fail_msg("Incorrect RssAnon value. Before: %ld After: %ld\n", 335 rss_anon_before, rss_anon_after); 336 } 337 338 static void split_pmd_zero_pages(void) 339 { 340 char *one_page; 341 int nr_hpages = 4; 342 size_t len = nr_hpages * pmd_pagesize; 343 344 one_page = allocate_zero_filled_hugepage(len); 345 verify_rss_anon_split_huge_page_all_zeroes(one_page, nr_hpages, len); 346 ksft_test_result_pass("Split zero filled huge pages successful\n"); 347 free(one_page); 348 } 349 350 static void split_pmd_thp_to_order(int order) 351 { 352 char *one_page; 353 size_t len = 4 * pmd_pagesize; 354 size_t i; 355 356 one_page = memalign(pmd_pagesize, len); 357 if (!one_page) 358 ksft_exit_fail_msg("Fail to allocate memory: %s\n", strerror(errno)); 359 360 madvise(one_page, len, MADV_HUGEPAGE); 361 362 for (i = 0; i < len; i++) 363 one_page[i] = (char)i; 364 365 if (!check_huge_anon(one_page, 4, pmd_pagesize)) 366 ksft_exit_fail_msg("No THP is allocated\n"); 367 368 /* split all THPs */ 369 write_debugfs(PID_FMT, getpid(), (uint64_t)one_page, 370 (uint64_t)one_page + len, order); 371 372 for (i = 0; i < len; i++) 373 if (one_page[i] != (char)i) 374 ksft_exit_fail_msg("%ld byte corrupted\n", i); 375 376 memset(expected_orders, 0, sizeof(int) * (pmd_order + 1)); 377 expected_orders[order] = 4 << (pmd_order - order); 378 379 if (check_after_split_folio_orders(one_page, len, pagemap_fd, 380 kpageflags_fd, expected_orders, 381 (pmd_order + 1))) 382 ksft_exit_fail_msg("Unexpected THP split\n"); 383 384 if (!check_huge_anon(one_page, 0, pmd_pagesize)) 385 ksft_exit_fail_msg("Still AnonHugePages not split\n"); 386 387 ksft_test_result_pass("Split huge pages to order %d successful\n", order); 388 free(one_page); 389 } 390 391 static void split_pte_mapped_thp(void) 392 { 393 char *one_page, *pte_mapped, *pte_mapped2; 394 size_t len = 4 * pmd_pagesize; 395 uint64_t thp_size; 396 size_t i; 397 398 one_page = mmap((void *)(1UL << 30), len, PROT_READ | PROT_WRITE, 399 MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); 400 if (one_page == MAP_FAILED) 401 ksft_exit_fail_msg("Fail to allocate memory: %s\n", strerror(errno)); 402 403 madvise(one_page, len, MADV_HUGEPAGE); 404 405 for (i = 0; i < len; i++) 406 one_page[i] = (char)i; 407 408 if (!check_huge_anon(one_page, 4, pmd_pagesize)) 409 ksft_exit_fail_msg("No THP is allocated\n"); 410 411 /* remap the first pagesize of first THP */ 412 pte_mapped = mremap(one_page, pagesize, pagesize, MREMAP_MAYMOVE); 413 414 /* remap the Nth pagesize of Nth THP */ 415 for (i = 1; i < 4; i++) { 416 pte_mapped2 = mremap(one_page + pmd_pagesize * i + pagesize * i, 417 pagesize, pagesize, 418 MREMAP_MAYMOVE|MREMAP_FIXED, 419 pte_mapped + pagesize * i); 420 if (pte_mapped2 == MAP_FAILED) 421 ksft_exit_fail_msg("mremap failed: %s\n", strerror(errno)); 422 } 423 424 /* smap does not show THPs after mremap, use kpageflags instead */ 425 thp_size = 0; 426 for (i = 0; i < pagesize * 4; i++) 427 if (i % pagesize == 0 && 428 is_backed_by_folio(&pte_mapped[i], pmd_order, pagemap_fd, kpageflags_fd)) 429 thp_size++; 430 431 if (thp_size != 4) 432 ksft_exit_fail_msg("Some THPs are missing during mremap\n"); 433 434 /* split all remapped THPs */ 435 write_debugfs(PID_FMT, getpid(), (uint64_t)pte_mapped, 436 (uint64_t)pte_mapped + pagesize * 4, 0); 437 438 /* smap does not show THPs after mremap, use kpageflags instead */ 439 thp_size = 0; 440 for (i = 0; i < pagesize * 4; i++) { 441 if (pte_mapped[i] != (char)i) 442 ksft_exit_fail_msg("%ld byte corrupted\n", i); 443 444 if (i % pagesize == 0 && 445 !is_backed_by_folio(&pte_mapped[i], 0, pagemap_fd, kpageflags_fd)) 446 thp_size++; 447 } 448 449 if (thp_size) 450 ksft_exit_fail_msg("Still %ld THPs not split\n", thp_size); 451 452 ksft_test_result_pass("Split PTE-mapped huge pages successful\n"); 453 munmap(one_page, len); 454 } 455 456 static void split_file_backed_thp(int order) 457 { 458 int status; 459 int fd; 460 char tmpfs_template[] = "/tmp/thp_split_XXXXXX"; 461 const char *tmpfs_loc = mkdtemp(tmpfs_template); 462 char testfile[INPUT_MAX]; 463 ssize_t num_written, num_read; 464 char *file_buf1, *file_buf2; 465 uint64_t pgoff_start = 0, pgoff_end = 1024; 466 int i; 467 468 ksft_print_msg("Please enable pr_debug in split_huge_pages_in_file() for more info.\n"); 469 470 file_buf1 = (char *)malloc(pmd_pagesize); 471 file_buf2 = (char *)malloc(pmd_pagesize); 472 473 if (!file_buf1 || !file_buf2) { 474 ksft_print_msg("cannot allocate file buffers\n"); 475 goto out; 476 } 477 478 for (i = 0; i < pmd_pagesize; i++) 479 file_buf1[i] = (char)i; 480 memset(file_buf2, 0, pmd_pagesize); 481 482 status = mount("tmpfs", tmpfs_loc, "tmpfs", 0, "huge=always,size=4m"); 483 484 if (status) 485 ksft_exit_fail_msg("Unable to create a tmpfs for testing\n"); 486 487 status = snprintf(testfile, INPUT_MAX, "%s/thp_file", tmpfs_loc); 488 if (status >= INPUT_MAX) { 489 ksft_exit_fail_msg("Fail to create file-backed THP split testing file\n"); 490 goto cleanup; 491 } 492 493 fd = open(testfile, O_CREAT|O_RDWR, 0664); 494 if (fd == -1) { 495 ksft_perror("Cannot open testing file"); 496 goto cleanup; 497 } 498 499 /* write pmd size data to the file, so a file-backed THP can be allocated */ 500 num_written = write(fd, file_buf1, pmd_pagesize); 501 502 if (num_written == -1 || num_written != pmd_pagesize) { 503 ksft_perror("Failed to write data to testing file"); 504 goto close_file; 505 } 506 507 /* split the file-backed THP */ 508 write_debugfs(PATH_FMT, testfile, pgoff_start, pgoff_end, order); 509 510 /* check file content after split */ 511 status = lseek(fd, 0, SEEK_SET); 512 if (status == -1) { 513 ksft_perror("Cannot lseek file"); 514 goto close_file; 515 } 516 517 num_read = read(fd, file_buf2, num_written); 518 if (num_read == -1 || num_read != num_written) { 519 ksft_perror("Cannot read file content back"); 520 goto close_file; 521 } 522 523 if (strncmp(file_buf1, file_buf2, pmd_pagesize) != 0) { 524 ksft_print_msg("File content changed\n"); 525 goto close_file; 526 } 527 528 close(fd); 529 status = unlink(testfile); 530 if (status) { 531 ksft_perror("Cannot remove testing file"); 532 goto cleanup; 533 } 534 535 status = umount(tmpfs_loc); 536 if (status) { 537 rmdir(tmpfs_loc); 538 ksft_exit_fail_msg("Unable to umount %s\n", tmpfs_loc); 539 } 540 541 status = rmdir(tmpfs_loc); 542 if (status) 543 ksft_exit_fail_msg("cannot remove tmp dir: %s\n", strerror(errno)); 544 545 ksft_print_msg("Please check dmesg for more information\n"); 546 ksft_test_result_pass("File-backed THP split to order %d test done\n", order); 547 return; 548 549 close_file: 550 close(fd); 551 cleanup: 552 umount(tmpfs_loc); 553 rmdir(tmpfs_loc); 554 out: 555 ksft_exit_fail_msg("Error occurred\n"); 556 } 557 558 static bool prepare_thp_fs(const char *xfs_path, char *thp_fs_template, 559 const char **thp_fs_loc) 560 { 561 if (xfs_path) { 562 *thp_fs_loc = xfs_path; 563 return false; 564 } 565 566 *thp_fs_loc = mkdtemp(thp_fs_template); 567 568 if (!*thp_fs_loc) 569 ksft_exit_fail_msg("cannot create temp folder\n"); 570 571 return true; 572 } 573 574 static void cleanup_thp_fs(const char *thp_fs_loc, bool created_tmp) 575 { 576 int status; 577 578 if (!created_tmp) 579 return; 580 581 status = rmdir(thp_fs_loc); 582 if (status) 583 ksft_exit_fail_msg("cannot remove tmp dir: %s\n", 584 strerror(errno)); 585 } 586 587 static int create_pagecache_thp_and_fd(const char *testfile, size_t fd_size, 588 int *fd, char **addr) 589 { 590 size_t i; 591 unsigned char buf[1024]; 592 593 srand(time(NULL)); 594 595 *fd = open(testfile, O_CREAT | O_RDWR, 0664); 596 if (*fd == -1) 597 ksft_exit_fail_msg("Failed to create a file at %s\n", testfile); 598 599 assert(fd_size % sizeof(buf) == 0); 600 for (i = 0; i < sizeof(buf); i++) 601 buf[i] = (unsigned char)i; 602 for (i = 0; i < fd_size; i += sizeof(buf)) 603 write(*fd, buf, sizeof(buf)); 604 605 close(*fd); 606 sync(); 607 *fd = open("/proc/sys/vm/drop_caches", O_WRONLY); 608 if (*fd == -1) { 609 ksft_perror("open drop_caches"); 610 goto err_out_unlink; 611 } 612 if (write(*fd, "3", 1) != 1) { 613 ksft_perror("write to drop_caches"); 614 goto err_out_unlink; 615 } 616 close(*fd); 617 618 *fd = open(testfile, O_RDWR); 619 if (*fd == -1) { 620 ksft_perror("Failed to open testfile\n"); 621 goto err_out_unlink; 622 } 623 624 *addr = mmap(NULL, fd_size, PROT_READ|PROT_WRITE, MAP_SHARED, *fd, 0); 625 if (*addr == (char *)-1) { 626 ksft_perror("cannot mmap"); 627 goto err_out_close; 628 } 629 madvise(*addr, fd_size, MADV_HUGEPAGE); 630 631 for (size_t i = 0; i < fd_size; i++) { 632 char *addr2 = *addr + i; 633 634 FORCE_READ(*addr2); 635 } 636 637 if (!check_huge_file(*addr, fd_size / pmd_pagesize, pmd_pagesize)) { 638 ksft_print_msg("No large pagecache folio generated, please provide a filesystem supporting large folio\n"); 639 munmap(*addr, fd_size); 640 close(*fd); 641 unlink(testfile); 642 ksft_test_result_skip("Pagecache folio split skipped\n"); 643 return -2; 644 } 645 return 0; 646 err_out_close: 647 close(*fd); 648 err_out_unlink: 649 unlink(testfile); 650 ksft_exit_fail_msg("Failed to create large pagecache folios\n"); 651 return -1; 652 } 653 654 static void split_thp_in_pagecache_to_order_at(size_t fd_size, 655 const char *fs_loc, int order, int offset) 656 { 657 int fd; 658 char *split_addr; 659 char *addr; 660 size_t i; 661 char testfile[INPUT_MAX]; 662 int err = 0; 663 664 err = snprintf(testfile, INPUT_MAX, "%s/test", fs_loc); 665 666 if (err < 0) 667 ksft_exit_fail_msg("cannot generate right test file name\n"); 668 669 err = create_pagecache_thp_and_fd(testfile, fd_size, &fd, &addr); 670 if (err) 671 return; 672 673 err = 0; 674 675 memset(expected_orders, 0, sizeof(int) * (pmd_order + 1)); 676 /* 677 * use [split_addr, split_addr + pagesize) range to split THPs, since 678 * the debugfs function always split a range with pagesize step and 679 * providing a full [addr, addr + fd_size) range can trigger multiple 680 * splits, complicating after-split result checking. 681 */ 682 if (offset == -1) { 683 for (split_addr = addr; split_addr < addr + fd_size; split_addr += pmd_pagesize) 684 write_debugfs(PID_FMT, getpid(), (uint64_t)split_addr, 685 (uint64_t)split_addr + pagesize, order); 686 687 expected_orders[order] = fd_size / (pagesize << order); 688 } else { 689 int times = fd_size / pmd_pagesize; 690 691 for (split_addr = addr; split_addr < addr + fd_size; split_addr += pmd_pagesize) 692 write_debugfs(PID_FMT_OFFSET, getpid(), (uint64_t)split_addr, 693 (uint64_t)split_addr + pagesize, order, offset); 694 695 for (i = order + 1; i < pmd_order; i++) 696 expected_orders[i] = times; 697 expected_orders[order] = 2 * times; 698 } 699 700 for (i = 0; i < fd_size; i++) 701 if (*(addr + i) != (char)i) { 702 ksft_print_msg("%lu byte corrupted in the file\n", i); 703 err = EXIT_FAILURE; 704 goto out; 705 } 706 707 if (check_after_split_folio_orders(addr, fd_size, pagemap_fd, 708 kpageflags_fd, expected_orders, 709 (pmd_order + 1))) { 710 ksft_print_msg("Unexpected THP split\n"); 711 err = 1; 712 goto out; 713 } 714 715 if (!check_huge_file(addr, 0, pmd_pagesize)) { 716 ksft_print_msg("Still FilePmdMapped not split\n"); 717 err = EXIT_FAILURE; 718 goto out; 719 } 720 721 out: 722 munmap(addr, fd_size); 723 close(fd); 724 unlink(testfile); 725 if (offset == -1) { 726 if (err) 727 ksft_exit_fail_msg("Split PMD-mapped pagecache folio to order %d failed\n", order); 728 ksft_test_result_pass("Split PMD-mapped pagecache folio to order %d passed\n", order); 729 } else { 730 if (err) 731 ksft_exit_fail_msg("Split PMD-mapped pagecache folio to order %d at in-folio offset %d failed\n", order, offset); 732 ksft_test_result_pass("Split PMD-mapped pagecache folio to order %d at in-folio offset %d passed\n", order, offset); 733 } 734 } 735 736 int main(int argc, char **argv) 737 { 738 int i; 739 size_t fd_size; 740 char *optional_xfs_path = NULL; 741 char fs_loc_template[] = "/tmp/thp_fs_XXXXXX"; 742 const char *fs_loc; 743 bool created_tmp; 744 int offset; 745 unsigned int nr_pages; 746 unsigned int tests; 747 748 ksft_print_header(); 749 750 if (geteuid() != 0) { 751 ksft_print_msg("Please run the benchmark as root\n"); 752 ksft_finished(); 753 } 754 755 if (argc > 1) 756 optional_xfs_path = argv[1]; 757 758 pagesize = getpagesize(); 759 pageshift = ffs(pagesize) - 1; 760 pmd_pagesize = read_pmd_pagesize(); 761 if (!pmd_pagesize) 762 ksft_exit_fail_msg("Reading PMD pagesize failed\n"); 763 764 nr_pages = pmd_pagesize / pagesize; 765 pmd_order = sz2ord(pmd_pagesize, pagesize); 766 767 expected_orders = (int *)malloc(sizeof(int) * (pmd_order + 1)); 768 if (!expected_orders) 769 ksft_exit_fail_msg("Fail to allocate memory: %s\n", strerror(errno)); 770 771 tests = 2 + (pmd_order - 1) + (2 * pmd_order) + (pmd_order - 1) * 4 + 2; 772 ksft_set_plan(tests); 773 774 pagemap_fd = open(pagemap_proc, O_RDONLY); 775 if (pagemap_fd == -1) 776 ksft_exit_fail_msg("read pagemap: %s\n", strerror(errno)); 777 778 kpageflags_fd = open(kpageflags_proc, O_RDONLY); 779 if (kpageflags_fd == -1) 780 ksft_exit_fail_msg("read kpageflags: %s\n", strerror(errno)); 781 782 fd_size = 2 * pmd_pagesize; 783 784 split_pmd_zero_pages(); 785 786 for (i = 0; i < pmd_order; i++) 787 if (i != 1) 788 split_pmd_thp_to_order(i); 789 790 split_pte_mapped_thp(); 791 for (i = 0; i < pmd_order; i++) 792 split_file_backed_thp(i); 793 794 created_tmp = prepare_thp_fs(optional_xfs_path, fs_loc_template, 795 &fs_loc); 796 for (i = pmd_order - 1; i >= 0; i--) 797 split_thp_in_pagecache_to_order_at(fd_size, fs_loc, i, -1); 798 799 for (i = 0; i < pmd_order; i++) 800 for (offset = 0; 801 offset < nr_pages; 802 offset += MAX(nr_pages / 4, 1 << i)) 803 split_thp_in_pagecache_to_order_at(fd_size, fs_loc, i, offset); 804 cleanup_thp_fs(fs_loc, created_tmp); 805 806 close(pagemap_fd); 807 close(kpageflags_fd); 808 free(expected_orders); 809 810 ksft_finished(); 811 812 return 0; 813 } 814