1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * A test of splitting PMD THPs and PTE-mapped THPs from a specified virtual 4 * address range in a process via <debugfs>/split_huge_pages interface. 5 */ 6 7 #define _GNU_SOURCE 8 #include <assert.h> 9 #include <stdio.h> 10 #include <stdlib.h> 11 #include <stdarg.h> 12 #include <unistd.h> 13 #include <inttypes.h> 14 #include <string.h> 15 #include <fcntl.h> 16 #include <sys/mman.h> 17 #include <sys/mount.h> 18 #include <sys/param.h> 19 #include <malloc.h> 20 #include <stdbool.h> 21 #include <time.h> 22 #include "vm_util.h" 23 #include "kselftest.h" 24 #include "hugepage_settings.h" 25 26 uint64_t pagesize; 27 unsigned int pageshift; 28 uint64_t pmd_pagesize; 29 unsigned int pmd_order; 30 int *expected_orders; 31 32 #define SPLIT_DEBUGFS "/sys/kernel/debug/split_huge_pages" 33 #define SMAP_PATH "/proc/self/smaps" 34 #define INPUT_MAX 80 35 36 #define PID_FMT "%d,0x%lx,0x%lx,%d" 37 #define PID_FMT_OFFSET "%d,0x%lx,0x%lx,%d,%d" 38 #define PATH_FMT "%s,0x%lx,0x%lx,%d" 39 40 const char *pagemap_proc = "/proc/self/pagemap"; 41 const char *kpageflags_proc = "/proc/kpageflags"; 42 int pagemap_fd; 43 int kpageflags_fd; 44 45 static bool is_backed_by_folio(char *vaddr, int order, int pagemap_fd, 46 int kpageflags_fd) 47 { 48 const uint64_t folio_head_flags = KPF_THP | KPF_COMPOUND_HEAD; 49 const uint64_t folio_tail_flags = KPF_THP | KPF_COMPOUND_TAIL; 50 const unsigned long nr_pages = 1UL << order; 51 unsigned long pfn_head; 52 uint64_t pfn_flags; 53 unsigned long pfn; 54 unsigned long i; 55 56 pfn = pagemap_get_pfn(pagemap_fd, vaddr); 57 58 /* non present page */ 59 if (pfn == -1UL) 60 return false; 61 62 if (pageflags_get(pfn, kpageflags_fd, &pfn_flags)) 63 goto fail; 64 65 /* check for order-0 pages */ 66 if (!order) { 67 if (pfn_flags & (folio_head_flags | folio_tail_flags)) 68 return false; 69 return true; 70 } 71 72 /* non THP folio */ 73 if (!(pfn_flags & KPF_THP)) 74 return false; 75 76 pfn_head = pfn & ~(nr_pages - 1); 77 78 if (pageflags_get(pfn_head, kpageflags_fd, &pfn_flags)) 79 goto fail; 80 81 /* head PFN has no compound_head flag set */ 82 if ((pfn_flags & folio_head_flags) != folio_head_flags) 83 return false; 84 85 /* check all tail PFN flags */ 86 for (i = 1; i < nr_pages; i++) { 87 if (pageflags_get(pfn_head + i, kpageflags_fd, &pfn_flags)) 88 goto fail; 89 if ((pfn_flags & folio_tail_flags) != folio_tail_flags) 90 return false; 91 } 92 93 /* 94 * check the PFN after this folio, but if its flags cannot be obtained, 95 * assume this folio has the expected order 96 */ 97 if (pageflags_get(pfn_head + nr_pages, kpageflags_fd, &pfn_flags)) 98 return true; 99 100 /* If we find another tail page, then the folio is larger. */ 101 return (pfn_flags & folio_tail_flags) != folio_tail_flags; 102 fail: 103 ksft_exit_fail_msg("Failed to get folio info\n"); 104 return false; 105 } 106 107 static int vaddr_pageflags_get(char *vaddr, int pagemap_fd, int kpageflags_fd, 108 uint64_t *flags) 109 { 110 unsigned long pfn; 111 112 pfn = pagemap_get_pfn(pagemap_fd, vaddr); 113 114 /* non-present PFN */ 115 if (pfn == -1UL) 116 return 1; 117 118 if (pageflags_get(pfn, kpageflags_fd, flags)) 119 return -1; 120 121 return 0; 122 } 123 124 /* 125 * gather_after_split_folio_orders - scan through [vaddr_start, len) and record 126 * folio orders 127 * 128 * @vaddr_start: start vaddr 129 * @len: range length 130 * @pagemap_fd: file descriptor to /proc/<pid>/pagemap 131 * @kpageflags_fd: file descriptor to /proc/kpageflags 132 * @orders: output folio order array 133 * @nr_orders: folio order array size 134 * 135 * gather_after_split_folio_orders() scan through [vaddr_start, len) and check 136 * all folios within the range and record their orders. All order-0 pages will 137 * be recorded. Non-present vaddr is skipped. 138 * 139 * NOTE: the function is used to check folio orders after a split is performed, 140 * so it assumes [vaddr_start, len) fully maps to after-split folios within that 141 * range. 142 * 143 * Return: 0 - no error, -1 - unhandled cases 144 */ 145 static int gather_after_split_folio_orders(char *vaddr_start, size_t len, 146 int pagemap_fd, int kpageflags_fd, int orders[], int nr_orders) 147 { 148 uint64_t page_flags = 0; 149 int cur_order = -1; 150 char *vaddr; 151 152 if (pagemap_fd == -1 || kpageflags_fd == -1) 153 return -1; 154 if (!orders) 155 return -1; 156 if (nr_orders <= 0) 157 return -1; 158 159 for (vaddr = vaddr_start; vaddr < vaddr_start + len;) { 160 char *next_folio_vaddr; 161 int status; 162 163 status = vaddr_pageflags_get(vaddr, pagemap_fd, kpageflags_fd, 164 &page_flags); 165 if (status < 0) 166 return -1; 167 168 /* skip non present vaddr */ 169 if (status == 1) { 170 vaddr += psize(); 171 continue; 172 } 173 174 /* all order-0 pages with possible false postive (non folio) */ 175 if (!(page_flags & (KPF_COMPOUND_HEAD | KPF_COMPOUND_TAIL))) { 176 orders[0]++; 177 vaddr += psize(); 178 continue; 179 } 180 181 /* skip non thp compound pages */ 182 if (!(page_flags & KPF_THP)) { 183 vaddr += psize(); 184 continue; 185 } 186 187 /* vpn points to part of a THP at this point */ 188 if (page_flags & KPF_COMPOUND_HEAD) 189 cur_order = 1; 190 else { 191 vaddr += psize(); 192 continue; 193 } 194 195 next_folio_vaddr = vaddr + (1UL << (cur_order + pshift())); 196 197 if (next_folio_vaddr >= vaddr_start + len) 198 break; 199 200 while ((status = vaddr_pageflags_get(next_folio_vaddr, 201 pagemap_fd, kpageflags_fd, 202 &page_flags)) >= 0) { 203 /* 204 * non present vaddr, next compound head page, or 205 * order-0 page 206 */ 207 if (status == 1 || 208 (page_flags & KPF_COMPOUND_HEAD) || 209 !(page_flags & (KPF_COMPOUND_HEAD | KPF_COMPOUND_TAIL))) { 210 if (cur_order < nr_orders) { 211 orders[cur_order]++; 212 cur_order = -1; 213 vaddr = next_folio_vaddr; 214 } 215 break; 216 } 217 218 cur_order++; 219 next_folio_vaddr = vaddr + (1UL << (cur_order + pshift())); 220 } 221 222 if (status < 0) 223 return status; 224 } 225 if (cur_order > 0 && cur_order < nr_orders) 226 orders[cur_order]++; 227 return 0; 228 } 229 230 static int check_after_split_folio_orders(char *vaddr_start, size_t len, 231 int pagemap_fd, int kpageflags_fd, int orders[], int nr_orders) 232 { 233 int *vaddr_orders; 234 int status; 235 int i; 236 237 vaddr_orders = (int *)malloc(sizeof(int) * nr_orders); 238 239 if (!vaddr_orders) 240 ksft_exit_fail_msg("Cannot allocate memory for vaddr_orders"); 241 242 memset(vaddr_orders, 0, sizeof(int) * nr_orders); 243 status = gather_after_split_folio_orders(vaddr_start, len, pagemap_fd, 244 kpageflags_fd, vaddr_orders, nr_orders); 245 if (status) 246 ksft_exit_fail_msg("gather folio info failed\n"); 247 248 for (i = 0; i < nr_orders; i++) 249 if (vaddr_orders[i] != orders[i]) { 250 ksft_print_msg("order %d: expected: %d got %d\n", i, 251 orders[i], vaddr_orders[i]); 252 status = -1; 253 } 254 255 free(vaddr_orders); 256 return status; 257 } 258 259 static void write_debugfs(const char *fmt, ...) 260 { 261 char input[INPUT_MAX]; 262 int ret; 263 va_list argp; 264 265 va_start(argp, fmt); 266 ret = vsnprintf(input, INPUT_MAX, fmt, argp); 267 va_end(argp); 268 269 if (ret >= INPUT_MAX) 270 ksft_exit_fail_msg("%s: Debugfs input is too long\n", __func__); 271 272 write_file(SPLIT_DEBUGFS, input, ret + 1); 273 } 274 275 static char *allocate_zero_filled_hugepage(size_t len) 276 { 277 char *result; 278 size_t i; 279 280 result = memalign(pmd_pagesize, len); 281 if (!result) { 282 printf("Fail to allocate memory\n"); 283 exit(EXIT_FAILURE); 284 } 285 286 madvise(result, len, MADV_HUGEPAGE); 287 288 for (i = 0; i < len; i++) 289 result[i] = (char)0; 290 291 return result; 292 } 293 294 static void verify_rss_anon_split_huge_page_all_zeroes(char *one_page, int nr_hpages, size_t len) 295 { 296 unsigned long rss_anon_before, rss_anon_after; 297 size_t i; 298 299 if (!check_huge_anon(one_page, nr_hpages, pmd_pagesize)) 300 ksft_exit_fail_msg("No THP is allocated\n"); 301 302 rss_anon_before = rss_anon(); 303 if (!rss_anon_before) 304 ksft_exit_fail_msg("No RssAnon is allocated before split\n"); 305 306 /* split all THPs */ 307 write_debugfs(PID_FMT, getpid(), (uint64_t)one_page, 308 (uint64_t)one_page + len, 0); 309 310 for (i = 0; i < len; i++) 311 if (one_page[i] != (char)0) 312 ksft_exit_fail_msg("%ld byte corrupted\n", i); 313 314 if (!check_huge_anon(one_page, 0, pmd_pagesize)) 315 ksft_exit_fail_msg("Still AnonHugePages not split\n"); 316 317 rss_anon_after = rss_anon(); 318 if (rss_anon_after >= rss_anon_before) 319 ksft_exit_fail_msg("Incorrect RssAnon value. Before: %ld After: %ld\n", 320 rss_anon_before, rss_anon_after); 321 } 322 323 static void split_pmd_zero_pages(void) 324 { 325 char *one_page; 326 int nr_hpages = 4; 327 size_t len = nr_hpages * pmd_pagesize; 328 329 one_page = allocate_zero_filled_hugepage(len); 330 verify_rss_anon_split_huge_page_all_zeroes(one_page, nr_hpages, len); 331 ksft_test_result_pass("Split zero filled huge pages successful\n"); 332 free(one_page); 333 } 334 335 static void split_pmd_thp_to_order(int order) 336 { 337 char *one_page; 338 size_t len = 4 * pmd_pagesize; 339 size_t i; 340 341 one_page = memalign(pmd_pagesize, len); 342 if (!one_page) 343 ksft_exit_fail_msg("Fail to allocate memory: %s\n", strerror(errno)); 344 345 madvise(one_page, len, MADV_HUGEPAGE); 346 347 for (i = 0; i < len; i++) 348 one_page[i] = (char)i; 349 350 if (!check_huge_anon(one_page, 4, pmd_pagesize)) 351 ksft_exit_fail_msg("No THP is allocated\n"); 352 353 /* split all THPs */ 354 write_debugfs(PID_FMT, getpid(), (uint64_t)one_page, 355 (uint64_t)one_page + len, order); 356 357 for (i = 0; i < len; i++) 358 if (one_page[i] != (char)i) 359 ksft_exit_fail_msg("%ld byte corrupted\n", i); 360 361 memset(expected_orders, 0, sizeof(int) * (pmd_order + 1)); 362 expected_orders[order] = 4 << (pmd_order - order); 363 364 if (check_after_split_folio_orders(one_page, len, pagemap_fd, 365 kpageflags_fd, expected_orders, 366 (pmd_order + 1))) 367 ksft_exit_fail_msg("Unexpected THP split\n"); 368 369 if (!check_huge_anon(one_page, 0, pmd_pagesize)) 370 ksft_exit_fail_msg("Still AnonHugePages not split\n"); 371 372 ksft_test_result_pass("Split huge pages to order %d successful\n", order); 373 free(one_page); 374 } 375 376 static void split_pte_mapped_thp(void) 377 { 378 const size_t nr_thps = 4; 379 const size_t thp_area_size = nr_thps * pmd_pagesize; 380 const size_t page_area_size = nr_thps * pagesize; 381 char *thp_area, *tmp, *page_area = MAP_FAILED; 382 size_t i; 383 384 thp_area = mmap((void *)(1UL << 30), thp_area_size, PROT_READ | PROT_WRITE, 385 MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); 386 if (thp_area == MAP_FAILED) { 387 ksft_test_result_fail("Fail to allocate memory: %s\n", strerror(errno)); 388 return; 389 } 390 391 madvise(thp_area, thp_area_size, MADV_HUGEPAGE); 392 393 for (i = 0; i < thp_area_size; i++) 394 thp_area[i] = (char)i; 395 396 if (!check_huge_anon(thp_area, nr_thps, pmd_pagesize)) { 397 ksft_test_result_skip("Not all THPs allocated\n"); 398 goto out; 399 } 400 401 /* 402 * To challenge spitting code, we will mremap a single page of each 403 * THP (page[i] of thp[i]) in the thp_area into page_area. This will 404 * replace the PMD mappings in the thp_area by PTE mappings first, 405 * but leaving the THP unsplit, to then create a page-sized hole in 406 * the thp_area. 407 * We will then manually trigger splitting of all THPs through the 408 * single mremap'ed pages of each THP in the page_area. 409 */ 410 page_area = mmap(NULL, page_area_size, PROT_READ | PROT_WRITE, 411 MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); 412 if (page_area == MAP_FAILED) { 413 ksft_test_result_fail("Fail to allocate memory: %s\n", strerror(errno)); 414 goto out; 415 } 416 417 for (i = 0; i < nr_thps; i++) { 418 tmp = mremap(thp_area + pmd_pagesize * i + pagesize * i, 419 pagesize, pagesize, MREMAP_MAYMOVE|MREMAP_FIXED, 420 page_area + pagesize * i); 421 if (tmp != MAP_FAILED) 422 continue; 423 ksft_test_result_fail("mremap failed: %s\n", strerror(errno)); 424 goto out; 425 } 426 427 /* 428 * Verify that our THPs were not split yet. Note that 429 * check_huge_anon() cannot be used as it checks for PMD mappings. 430 */ 431 for (i = 0; i < nr_thps; i++) { 432 if (is_backed_by_folio(page_area + i * pagesize, pmd_order, 433 pagemap_fd, kpageflags_fd)) 434 continue; 435 ksft_test_result_fail("THP %zu missing after mremap\n", i); 436 goto out; 437 } 438 439 /* Split all THPs through the remapped pages. */ 440 write_debugfs(PID_FMT, getpid(), (uint64_t)page_area, 441 (uint64_t)page_area + page_area_size, 0); 442 443 /* Corruption during mremap or split? */ 444 for (i = 0; i < page_area_size; i++) { 445 if (page_area[i] == (char)i) 446 continue; 447 ksft_test_result_fail("%zu byte corrupted\n", i); 448 goto out; 449 } 450 451 /* Split failed? */ 452 for (i = 0; i < nr_thps; i++) { 453 if (is_backed_by_folio(page_area + i * pagesize, 0, 454 pagemap_fd, kpageflags_fd)) 455 continue; 456 ksft_test_result_fail("THP %zu not split\n", i); 457 } 458 459 ksft_test_result_pass("Split PTE-mapped huge pages successful\n"); 460 out: 461 munmap(thp_area, thp_area_size); 462 if (page_area != MAP_FAILED) 463 munmap(page_area, page_area_size); 464 } 465 466 static void split_file_backed_thp(int order) 467 { 468 int status; 469 int fd; 470 char tmpfs_template[] = "/tmp/thp_split_XXXXXX"; 471 const char *tmpfs_loc = mkdtemp(tmpfs_template); 472 char testfile[INPUT_MAX]; 473 unsigned long size = 2 * pmd_pagesize; 474 char opts[64]; 475 ssize_t num_written, num_read; 476 char *file_buf1 = NULL, *file_buf2 = NULL; 477 uint64_t pgoff_start = 0, pgoff_end = 1024; 478 int i; 479 480 ksft_print_msg("Please enable pr_debug in split_huge_pages_in_file() for more info.\n"); 481 482 if (!tmpfs_loc) 483 ksft_exit_fail_msg("mkdtemp failed\n"); 484 485 file_buf1 = (char *)malloc(pmd_pagesize); 486 file_buf2 = (char *)malloc(pmd_pagesize); 487 488 if (!file_buf1 || !file_buf2) { 489 ksft_print_msg("cannot allocate file buffers\n"); 490 goto out; 491 } 492 493 for (i = 0; i < pmd_pagesize; i++) 494 file_buf1[i] = (char)i; 495 memset(file_buf2, 0, pmd_pagesize); 496 497 snprintf(opts, sizeof(opts), "huge=always,size=%lu", size); 498 status = mount("tmpfs", tmpfs_loc, "tmpfs", 0, opts); 499 500 if (status) { 501 ksft_print_msg("Unable to create a tmpfs for testing\n"); 502 goto out; 503 } 504 505 status = snprintf(testfile, INPUT_MAX, "%s/thp_file", tmpfs_loc); 506 if (status >= INPUT_MAX) { 507 ksft_print_msg("Fail to create file-backed THP split testing file\n"); 508 goto cleanup; 509 } 510 511 fd = open(testfile, O_CREAT|O_RDWR, 0664); 512 if (fd == -1) { 513 ksft_perror("Cannot open testing file"); 514 goto cleanup; 515 } 516 517 /* write pmd size data to the file, so a file-backed THP can be allocated */ 518 num_written = write(fd, file_buf1, pmd_pagesize); 519 520 if (num_written == -1 || num_written != pmd_pagesize) { 521 ksft_perror("Failed to write data to testing file"); 522 goto close_file; 523 } 524 525 /* split the file-backed THP */ 526 write_debugfs(PATH_FMT, testfile, pgoff_start, pgoff_end, order); 527 528 /* check file content after split */ 529 status = lseek(fd, 0, SEEK_SET); 530 if (status == -1) { 531 ksft_perror("Cannot lseek file"); 532 goto close_file; 533 } 534 535 num_read = read(fd, file_buf2, num_written); 536 if (num_read == -1 || num_read != num_written) { 537 ksft_perror("Cannot read file content back"); 538 goto close_file; 539 } 540 541 if (strncmp(file_buf1, file_buf2, pmd_pagesize) != 0) { 542 ksft_print_msg("File content changed\n"); 543 goto close_file; 544 } 545 546 close(fd); 547 status = unlink(testfile); 548 if (status) { 549 ksft_perror("Cannot remove testing file"); 550 goto cleanup; 551 } 552 553 status = umount(tmpfs_loc); 554 if (status) { 555 ksft_print_msg("Unable to umount %s\n", tmpfs_loc); 556 goto out; 557 } 558 559 free(file_buf1); 560 free(file_buf2); 561 562 status = rmdir(tmpfs_loc); 563 if (status) 564 ksft_exit_fail_msg("cannot remove tmp dir: %s\n", strerror(errno)); 565 566 ksft_print_msg("Please check dmesg for more information\n"); 567 ksft_test_result_pass("File-backed THP split to order %d test done\n", order); 568 return; 569 570 close_file: 571 close(fd); 572 cleanup: 573 umount(tmpfs_loc); 574 out: 575 free(file_buf1); 576 free(file_buf2); 577 rmdir(tmpfs_loc); 578 ksft_exit_fail_msg("Error occurred\n"); 579 } 580 581 static bool prepare_thp_fs(const char *xfs_path, char *thp_fs_template, 582 const char **thp_fs_loc) 583 { 584 if (xfs_path) { 585 *thp_fs_loc = xfs_path; 586 return false; 587 } 588 589 *thp_fs_loc = mkdtemp(thp_fs_template); 590 591 if (!*thp_fs_loc) 592 ksft_exit_fail_msg("cannot create temp folder\n"); 593 594 return true; 595 } 596 597 static void cleanup_thp_fs(const char *thp_fs_loc, bool created_tmp) 598 { 599 int status; 600 601 if (!created_tmp) 602 return; 603 604 status = rmdir(thp_fs_loc); 605 if (status) 606 ksft_exit_fail_msg("cannot remove tmp dir: %s\n", 607 strerror(errno)); 608 } 609 610 static int create_pagecache_thp_and_fd(const char *testfile, size_t fd_size, 611 int *fd, char **addr) 612 { 613 size_t i; 614 unsigned char buf[1024]; 615 616 srand(time(NULL)); 617 618 *fd = open(testfile, O_CREAT | O_RDWR, 0664); 619 if (*fd == -1) 620 ksft_exit_fail_msg("Failed to create a file at %s\n", testfile); 621 622 assert(fd_size % sizeof(buf) == 0); 623 for (i = 0; i < sizeof(buf); i++) 624 buf[i] = (unsigned char)i; 625 for (i = 0; i < fd_size; i += sizeof(buf)) { 626 if (write(*fd, buf, sizeof(buf)) != sizeof(buf)) { 627 ksft_perror("write testfile"); 628 close(*fd); 629 goto err_out_unlink; 630 } 631 } 632 close(*fd); 633 sync(); 634 *fd = open("/proc/sys/vm/drop_caches", O_WRONLY); 635 if (*fd == -1) { 636 ksft_perror("open drop_caches"); 637 goto err_out_unlink; 638 } 639 if (write(*fd, "3", 1) != 1) { 640 ksft_perror("write to drop_caches"); 641 goto err_out_close; 642 } 643 close(*fd); 644 645 *fd = open(testfile, O_RDWR); 646 if (*fd == -1) { 647 ksft_perror("Failed to open testfile\n"); 648 goto err_out_unlink; 649 } 650 651 *addr = mmap(NULL, fd_size, PROT_READ|PROT_WRITE, MAP_SHARED, *fd, 0); 652 if (*addr == (char *)-1) { 653 ksft_perror("cannot mmap"); 654 goto err_out_close; 655 } 656 madvise(*addr, fd_size, MADV_HUGEPAGE); 657 658 force_read_pages(*addr, fd_size / pmd_pagesize, pmd_pagesize); 659 660 if (!check_huge_file(*addr, fd_size / pmd_pagesize, pmd_pagesize)) { 661 ksft_print_msg("No large pagecache folio generated, please provide a filesystem supporting large folio\n"); 662 munmap(*addr, fd_size); 663 close(*fd); 664 unlink(testfile); 665 ksft_test_result_skip("Pagecache folio split skipped\n"); 666 return -2; 667 } 668 return 0; 669 err_out_close: 670 close(*fd); 671 err_out_unlink: 672 unlink(testfile); 673 ksft_exit_fail_msg("Failed to create large pagecache folios\n"); 674 return -1; 675 } 676 677 static void split_thp_in_pagecache_to_order_at(size_t fd_size, 678 const char *fs_loc, int order, int offset) 679 { 680 int fd; 681 char *split_addr; 682 char *addr; 683 size_t i; 684 char testfile[INPUT_MAX]; 685 int err = 0; 686 687 err = snprintf(testfile, INPUT_MAX, "%s/test", fs_loc); 688 689 if (err < 0) 690 ksft_exit_fail_msg("cannot generate right test file name\n"); 691 692 err = create_pagecache_thp_and_fd(testfile, fd_size, &fd, &addr); 693 if (err) 694 return; 695 696 err = 0; 697 698 memset(expected_orders, 0, sizeof(int) * (pmd_order + 1)); 699 /* 700 * use [split_addr, split_addr + pagesize) range to split THPs, since 701 * the debugfs function always split a range with pagesize step and 702 * providing a full [addr, addr + fd_size) range can trigger multiple 703 * splits, complicating after-split result checking. 704 */ 705 if (offset == -1) { 706 for (split_addr = addr; split_addr < addr + fd_size; split_addr += pmd_pagesize) 707 write_debugfs(PID_FMT, getpid(), (uint64_t)split_addr, 708 (uint64_t)split_addr + pagesize, order); 709 710 expected_orders[order] = fd_size / (pagesize << order); 711 } else { 712 int times = fd_size / pmd_pagesize; 713 714 for (split_addr = addr; split_addr < addr + fd_size; split_addr += pmd_pagesize) 715 write_debugfs(PID_FMT_OFFSET, getpid(), (uint64_t)split_addr, 716 (uint64_t)split_addr + pagesize, order, offset); 717 718 for (i = order + 1; i < pmd_order; i++) 719 expected_orders[i] = times; 720 expected_orders[order] = 2 * times; 721 } 722 723 for (i = 0; i < fd_size; i++) 724 if (*(addr + i) != (char)i) { 725 ksft_print_msg("%lu byte corrupted in the file\n", i); 726 err = EXIT_FAILURE; 727 goto out; 728 } 729 730 if (check_after_split_folio_orders(addr, fd_size, pagemap_fd, 731 kpageflags_fd, expected_orders, 732 (pmd_order + 1))) { 733 ksft_print_msg("Unexpected THP split\n"); 734 err = 1; 735 goto out; 736 } 737 738 if (!check_huge_file(addr, 0, pmd_pagesize)) { 739 ksft_print_msg("Still FilePmdMapped not split\n"); 740 err = EXIT_FAILURE; 741 goto out; 742 } 743 744 out: 745 munmap(addr, fd_size); 746 close(fd); 747 unlink(testfile); 748 if (offset == -1) { 749 if (err) 750 ksft_exit_fail_msg("Split PMD-mapped pagecache folio to order %d failed\n", order); 751 ksft_test_result_pass("Split PMD-mapped pagecache folio to order %d passed\n", order); 752 } else { 753 if (err) 754 ksft_exit_fail_msg("Split PMD-mapped pagecache folio to order %d at in-folio offset %d failed\n", order, offset); 755 ksft_test_result_pass("Split PMD-mapped pagecache folio to order %d at in-folio offset %d passed\n", order, offset); 756 } 757 } 758 759 int main(int argc, char **argv) 760 { 761 int i; 762 size_t fd_size; 763 char *optional_xfs_path = NULL; 764 char fs_loc_template[] = "/tmp/thp_fs_XXXXXX"; 765 const char *fs_loc; 766 bool created_tmp; 767 int offset; 768 unsigned int nr_pages; 769 unsigned int tests; 770 771 ksft_print_header(); 772 773 if (geteuid() != 0) { 774 ksft_print_msg("Please run the benchmark as root\n"); 775 ksft_finished(); 776 } 777 778 if (!thp_is_enabled()) 779 ksft_exit_skip("Transparent Hugepages not available\n"); 780 781 if (argc > 1) 782 optional_xfs_path = argv[1]; 783 784 pagesize = getpagesize(); 785 pageshift = ffs(pagesize) - 1; 786 pmd_pagesize = read_pmd_pagesize(); 787 if (!pmd_pagesize) 788 ksft_exit_fail_msg("Reading PMD pagesize failed\n"); 789 790 nr_pages = pmd_pagesize / pagesize; 791 pmd_order = sz2ord(pmd_pagesize, pagesize); 792 793 expected_orders = (int *)malloc(sizeof(int) * (pmd_order + 1)); 794 if (!expected_orders) 795 ksft_exit_fail_msg("Fail to allocate memory: %s\n", strerror(errno)); 796 797 tests = 2 + (pmd_order - 1) + (2 * pmd_order) + (pmd_order - 1) * 4 + 2; 798 ksft_set_plan(tests); 799 800 pagemap_fd = open(pagemap_proc, O_RDONLY); 801 if (pagemap_fd == -1) 802 ksft_exit_fail_msg("read pagemap: %s\n", strerror(errno)); 803 804 kpageflags_fd = open(kpageflags_proc, O_RDONLY); 805 if (kpageflags_fd == -1) 806 ksft_exit_fail_msg("read kpageflags: %s\n", strerror(errno)); 807 808 fd_size = 2 * pmd_pagesize; 809 810 split_pmd_zero_pages(); 811 812 for (i = 0; i < pmd_order; i++) 813 if (i != 1) 814 split_pmd_thp_to_order(i); 815 816 split_pte_mapped_thp(); 817 for (i = 0; i < pmd_order; i++) 818 split_file_backed_thp(i); 819 820 created_tmp = prepare_thp_fs(optional_xfs_path, fs_loc_template, 821 &fs_loc); 822 for (i = pmd_order - 1; i >= 0; i--) 823 split_thp_in_pagecache_to_order_at(fd_size, fs_loc, i, -1); 824 825 for (i = 0; i < pmd_order; i++) 826 for (offset = 0; 827 offset < nr_pages; 828 offset += MAX(nr_pages / 4, 1 << i)) 829 split_thp_in_pagecache_to_order_at(fd_size, fs_loc, i, offset); 830 cleanup_thp_fs(fs_loc, created_tmp); 831 832 close(pagemap_fd); 833 close(kpageflags_fd); 834 free(expected_orders); 835 836 ksft_finished(); 837 838 return 0; 839 } 840