1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * A test of splitting PMD THPs and PTE-mapped THPs from a specified virtual 4 * address range in a process via <debugfs>/split_huge_pages interface. 5 */ 6 7 #define _GNU_SOURCE 8 #include <assert.h> 9 #include <stdio.h> 10 #include <stdlib.h> 11 #include <stdarg.h> 12 #include <unistd.h> 13 #include <inttypes.h> 14 #include <string.h> 15 #include <fcntl.h> 16 #include <sys/mman.h> 17 #include <sys/mount.h> 18 #include <sys/param.h> 19 #include <malloc.h> 20 #include <stdbool.h> 21 #include <time.h> 22 #include "vm_util.h" 23 #include "kselftest.h" 24 #include "hugepage_settings.h" 25 26 uint64_t pagesize; 27 unsigned int pageshift; 28 uint64_t pmd_pagesize; 29 unsigned int pmd_order; 30 int *expected_orders; 31 32 #define SPLIT_DEBUGFS "/sys/kernel/debug/split_huge_pages" 33 #define SMAP_PATH "/proc/self/smaps" 34 #define INPUT_MAX 80 35 36 #define PID_FMT "%d,0x%lx,0x%lx,%d" 37 #define PID_FMT_OFFSET "%d,0x%lx,0x%lx,%d,%d" 38 #define PATH_FMT "%s,0x%lx,0x%lx,%d" 39 40 const char *pagemap_proc = "/proc/self/pagemap"; 41 const char *kpageflags_proc = "/proc/kpageflags"; 42 int pagemap_fd; 43 int kpageflags_fd; 44 45 static bool is_backed_by_folio(char *vaddr, int order, int pagemap_fd, 46 int kpageflags_fd) 47 { 48 const uint64_t folio_head_flags = KPF_THP | KPF_COMPOUND_HEAD; 49 const uint64_t folio_tail_flags = KPF_THP | KPF_COMPOUND_TAIL; 50 const unsigned long nr_pages = 1UL << order; 51 unsigned long pfn_head; 52 uint64_t pfn_flags; 53 unsigned long pfn; 54 unsigned long i; 55 56 pfn = pagemap_get_pfn(pagemap_fd, vaddr); 57 58 /* non present page */ 59 if (pfn == -1UL) 60 return false; 61 62 if (pageflags_get(pfn, kpageflags_fd, &pfn_flags)) 63 goto fail; 64 65 /* check for order-0 pages */ 66 if (!order) { 67 if (pfn_flags & (folio_head_flags | folio_tail_flags)) 68 return false; 69 return true; 70 } 71 72 /* non THP folio */ 73 if (!(pfn_flags & KPF_THP)) 74 return false; 75 76 pfn_head = pfn & ~(nr_pages - 1); 77 78 if (pageflags_get(pfn_head, kpageflags_fd, &pfn_flags)) 79 goto fail; 80 81 /* head PFN has no compound_head flag set */ 82 if ((pfn_flags & folio_head_flags) != folio_head_flags) 83 return false; 84 85 /* check all tail PFN flags */ 86 for (i = 1; i < nr_pages; i++) { 87 if (pageflags_get(pfn_head + i, kpageflags_fd, &pfn_flags)) 88 goto fail; 89 if ((pfn_flags & folio_tail_flags) != folio_tail_flags) 90 return false; 91 } 92 93 /* 94 * check the PFN after this folio, but if its flags cannot be obtained, 95 * assume this folio has the expected order 96 */ 97 if (pageflags_get(pfn_head + nr_pages, kpageflags_fd, &pfn_flags)) 98 return true; 99 100 /* If we find another tail page, then the folio is larger. */ 101 return (pfn_flags & folio_tail_flags) != folio_tail_flags; 102 fail: 103 ksft_exit_fail_msg("Failed to get folio info\n"); 104 return false; 105 } 106 107 static int check_after_split_folio_orders(char *vaddr_start, size_t len, 108 int pagemap_fd, int kpageflags_fd, int orders[], int nr_orders) 109 { 110 int *vaddr_orders; 111 int status; 112 int i; 113 114 vaddr_orders = (int *)malloc(sizeof(int) * nr_orders); 115 116 if (!vaddr_orders) 117 ksft_exit_fail_msg("Cannot allocate memory for vaddr_orders"); 118 119 memset(vaddr_orders, 0, sizeof(int) * nr_orders); 120 status = gather_folio_orders(vaddr_start, len, pagemap_fd, 121 kpageflags_fd, vaddr_orders, nr_orders); 122 if (status) 123 ksft_exit_fail_msg("gather folio info failed\n"); 124 125 for (i = 0; i < nr_orders; i++) 126 if (vaddr_orders[i] != orders[i]) { 127 ksft_print_msg("order %d: expected: %d got %d\n", i, 128 orders[i], vaddr_orders[i]); 129 status = -1; 130 } 131 132 free(vaddr_orders); 133 return status; 134 } 135 136 static void write_debugfs(const char *fmt, ...) 137 { 138 char input[INPUT_MAX]; 139 int ret; 140 va_list argp; 141 142 va_start(argp, fmt); 143 ret = vsnprintf(input, INPUT_MAX, fmt, argp); 144 va_end(argp); 145 146 if (ret >= INPUT_MAX) 147 ksft_exit_fail_msg("%s: Debugfs input is too long\n", __func__); 148 149 write_file(SPLIT_DEBUGFS, input, ret + 1); 150 } 151 152 static char *allocate_zero_filled_hugepage(size_t len) 153 { 154 char *result; 155 size_t i; 156 157 result = memalign(pmd_pagesize, len); 158 if (!result) { 159 printf("Fail to allocate memory\n"); 160 exit(EXIT_FAILURE); 161 } 162 163 madvise(result, len, MADV_HUGEPAGE); 164 165 for (i = 0; i < len; i++) 166 result[i] = (char)0; 167 168 return result; 169 } 170 171 static void verify_rss_anon_split_huge_page_all_zeroes(char *one_page, int nr_hpages, size_t len) 172 { 173 unsigned long rss_anon_before, rss_anon_after; 174 size_t i; 175 176 if (!check_huge_anon(one_page, nr_hpages * pmd_pagesize, nr_hpages, pmd_pagesize)) 177 ksft_exit_fail_msg("No THP is allocated\n"); 178 179 rss_anon_before = rss_anon(); 180 if (!rss_anon_before) 181 ksft_exit_fail_msg("No RssAnon is allocated before split\n"); 182 183 /* split all THPs */ 184 write_debugfs(PID_FMT, getpid(), (uint64_t)one_page, 185 (uint64_t)one_page + len, 0); 186 187 for (i = 0; i < len; i++) 188 if (one_page[i] != (char)0) 189 ksft_exit_fail_msg("%ld byte corrupted\n", i); 190 191 if (!check_huge_anon(one_page, nr_hpages * pmd_pagesize, 0, pmd_pagesize)) 192 ksft_exit_fail_msg("Still AnonHugePages not split\n"); 193 194 rss_anon_after = rss_anon(); 195 if (rss_anon_after >= rss_anon_before) 196 ksft_exit_fail_msg("Incorrect RssAnon value. Before: %ld After: %ld\n", 197 rss_anon_before, rss_anon_after); 198 } 199 200 static void split_pmd_zero_pages(void) 201 { 202 char *one_page; 203 int nr_hpages = 4; 204 size_t len = nr_hpages * pmd_pagesize; 205 206 one_page = allocate_zero_filled_hugepage(len); 207 verify_rss_anon_split_huge_page_all_zeroes(one_page, nr_hpages, len); 208 ksft_test_result_pass("Split zero filled huge pages successful\n"); 209 free(one_page); 210 } 211 212 static void split_pmd_thp_to_order(int order) 213 { 214 char *one_page; 215 size_t len = 4 * pmd_pagesize; 216 size_t i; 217 218 one_page = memalign(pmd_pagesize, len); 219 if (!one_page) 220 ksft_exit_fail_msg("Fail to allocate memory: %s\n", strerror(errno)); 221 222 madvise(one_page, len, MADV_HUGEPAGE); 223 224 for (i = 0; i < len; i++) 225 one_page[i] = (char)i; 226 227 if (!check_huge_anon(one_page, 4 * pmd_pagesize, 4, pmd_pagesize)) 228 ksft_exit_fail_msg("No THP is allocated\n"); 229 230 /* split all THPs */ 231 write_debugfs(PID_FMT, getpid(), (uint64_t)one_page, 232 (uint64_t)one_page + len, order); 233 234 for (i = 0; i < len; i++) 235 if (one_page[i] != (char)i) 236 ksft_exit_fail_msg("%ld byte corrupted\n", i); 237 238 memset(expected_orders, 0, sizeof(int) * (pmd_order + 1)); 239 expected_orders[order] = 4 << (pmd_order - order); 240 241 if (check_after_split_folio_orders(one_page, len, pagemap_fd, 242 kpageflags_fd, expected_orders, 243 (pmd_order + 1))) 244 ksft_exit_fail_msg("Unexpected THP split\n"); 245 246 if (!check_huge_anon(one_page, 4 * pmd_pagesize, 0, pmd_pagesize)) 247 ksft_exit_fail_msg("Still AnonHugePages not split\n"); 248 249 ksft_test_result_pass("Split huge pages to order %d successful\n", order); 250 free(one_page); 251 } 252 253 static void split_pte_mapped_thp(void) 254 { 255 const size_t nr_thps = 4; 256 const size_t thp_area_size = nr_thps * pmd_pagesize; 257 const size_t page_area_size = nr_thps * pagesize; 258 char *thp_area, *tmp, *page_area = MAP_FAILED; 259 size_t i; 260 261 thp_area = mmap((void *)(1UL << 30), thp_area_size, PROT_READ | PROT_WRITE, 262 MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); 263 if (thp_area == MAP_FAILED) { 264 ksft_test_result_fail("Fail to allocate memory: %s\n", strerror(errno)); 265 return; 266 } 267 268 madvise(thp_area, thp_area_size, MADV_HUGEPAGE); 269 270 for (i = 0; i < thp_area_size; i++) 271 thp_area[i] = (char)i; 272 273 if (!check_huge_anon(thp_area, nr_thps * pmd_pagesize, nr_thps, pmd_pagesize)) { 274 ksft_test_result_skip("Not all THPs allocated\n"); 275 goto out; 276 } 277 278 /* 279 * To challenge spitting code, we will mremap a single page of each 280 * THP (page[i] of thp[i]) in the thp_area into page_area. This will 281 * replace the PMD mappings in the thp_area by PTE mappings first, 282 * but leaving the THP unsplit, to then create a page-sized hole in 283 * the thp_area. 284 * We will then manually trigger splitting of all THPs through the 285 * single mremap'ed pages of each THP in the page_area. 286 */ 287 page_area = mmap(NULL, page_area_size, PROT_READ | PROT_WRITE, 288 MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); 289 if (page_area == MAP_FAILED) { 290 ksft_test_result_fail("Fail to allocate memory: %s\n", strerror(errno)); 291 goto out; 292 } 293 294 for (i = 0; i < nr_thps; i++) { 295 tmp = mremap(thp_area + pmd_pagesize * i + pagesize * i, 296 pagesize, pagesize, MREMAP_MAYMOVE|MREMAP_FIXED, 297 page_area + pagesize * i); 298 if (tmp != MAP_FAILED) 299 continue; 300 ksft_test_result_fail("mremap failed: %s\n", strerror(errno)); 301 goto out; 302 } 303 304 /* 305 * Verify that our THPs were not split yet. Note that 306 * check_huge_anon() cannot be used as it checks for PMD mappings. 307 */ 308 for (i = 0; i < nr_thps; i++) { 309 if (is_backed_by_folio(page_area + i * pagesize, pmd_order, 310 pagemap_fd, kpageflags_fd)) 311 continue; 312 ksft_test_result_fail("THP %zu missing after mremap\n", i); 313 goto out; 314 } 315 316 /* Split all THPs through the remapped pages. */ 317 write_debugfs(PID_FMT, getpid(), (uint64_t)page_area, 318 (uint64_t)page_area + page_area_size, 0); 319 320 /* Corruption during mremap or split? */ 321 for (i = 0; i < page_area_size; i++) { 322 if (page_area[i] == (char)i) 323 continue; 324 ksft_test_result_fail("%zu byte corrupted\n", i); 325 goto out; 326 } 327 328 /* Split failed? */ 329 for (i = 0; i < nr_thps; i++) { 330 if (is_backed_by_folio(page_area + i * pagesize, 0, 331 pagemap_fd, kpageflags_fd)) 332 continue; 333 ksft_test_result_fail("THP %zu not split\n", i); 334 } 335 336 ksft_test_result_pass("Split PTE-mapped huge pages successful\n"); 337 out: 338 munmap(thp_area, thp_area_size); 339 if (page_area != MAP_FAILED) 340 munmap(page_area, page_area_size); 341 } 342 343 static void split_file_backed_thp(int order) 344 { 345 int status; 346 int fd; 347 char tmpfs_template[] = "/tmp/thp_split_XXXXXX"; 348 const char *tmpfs_loc = mkdtemp(tmpfs_template); 349 char testfile[INPUT_MAX]; 350 unsigned long size = 2 * pmd_pagesize; 351 char opts[64]; 352 ssize_t num_written, num_read; 353 char *file_buf1 = NULL, *file_buf2 = NULL; 354 uint64_t pgoff_start = 0, pgoff_end = 1024; 355 int i; 356 357 ksft_print_msg("Please enable pr_debug in split_huge_pages_in_file() for more info.\n"); 358 359 if (!tmpfs_loc) 360 ksft_exit_fail_msg("mkdtemp failed\n"); 361 362 file_buf1 = (char *)malloc(pmd_pagesize); 363 file_buf2 = (char *)malloc(pmd_pagesize); 364 365 if (!file_buf1 || !file_buf2) { 366 ksft_print_msg("cannot allocate file buffers\n"); 367 goto out; 368 } 369 370 for (i = 0; i < pmd_pagesize; i++) 371 file_buf1[i] = (char)i; 372 memset(file_buf2, 0, pmd_pagesize); 373 374 snprintf(opts, sizeof(opts), "huge=always,size=%lu", size); 375 status = mount("tmpfs", tmpfs_loc, "tmpfs", 0, opts); 376 377 if (status) { 378 ksft_print_msg("Unable to create a tmpfs for testing\n"); 379 goto out; 380 } 381 382 status = snprintf(testfile, INPUT_MAX, "%s/thp_file", tmpfs_loc); 383 if (status >= INPUT_MAX) { 384 ksft_print_msg("Fail to create file-backed THP split testing file\n"); 385 goto cleanup; 386 } 387 388 fd = open(testfile, O_CREAT|O_RDWR, 0664); 389 if (fd == -1) { 390 ksft_perror("Cannot open testing file"); 391 goto cleanup; 392 } 393 394 /* write pmd size data to the file, so a file-backed THP can be allocated */ 395 num_written = write(fd, file_buf1, pmd_pagesize); 396 397 if (num_written == -1 || num_written != pmd_pagesize) { 398 ksft_perror("Failed to write data to testing file"); 399 goto close_file; 400 } 401 402 /* split the file-backed THP */ 403 write_debugfs(PATH_FMT, testfile, pgoff_start, pgoff_end, order); 404 405 /* check file content after split */ 406 status = lseek(fd, 0, SEEK_SET); 407 if (status == -1) { 408 ksft_perror("Cannot lseek file"); 409 goto close_file; 410 } 411 412 num_read = read(fd, file_buf2, num_written); 413 if (num_read == -1 || num_read != num_written) { 414 ksft_perror("Cannot read file content back"); 415 goto close_file; 416 } 417 418 if (strncmp(file_buf1, file_buf2, pmd_pagesize) != 0) { 419 ksft_print_msg("File content changed\n"); 420 goto close_file; 421 } 422 423 close(fd); 424 status = unlink(testfile); 425 if (status) { 426 ksft_perror("Cannot remove testing file"); 427 goto cleanup; 428 } 429 430 status = umount(tmpfs_loc); 431 if (status) { 432 ksft_print_msg("Unable to umount %s\n", tmpfs_loc); 433 goto out; 434 } 435 436 free(file_buf1); 437 free(file_buf2); 438 439 status = rmdir(tmpfs_loc); 440 if (status) 441 ksft_exit_fail_msg("cannot remove tmp dir: %s\n", strerror(errno)); 442 443 ksft_print_msg("Please check dmesg for more information\n"); 444 ksft_test_result_pass("File-backed THP split to order %d test done\n", order); 445 return; 446 447 close_file: 448 close(fd); 449 cleanup: 450 umount(tmpfs_loc); 451 out: 452 free(file_buf1); 453 free(file_buf2); 454 rmdir(tmpfs_loc); 455 ksft_exit_fail_msg("Error occurred\n"); 456 } 457 458 static bool prepare_thp_fs(const char *xfs_path, char *thp_fs_template, 459 const char **thp_fs_loc) 460 { 461 if (xfs_path) { 462 *thp_fs_loc = xfs_path; 463 return false; 464 } 465 466 *thp_fs_loc = mkdtemp(thp_fs_template); 467 468 if (!*thp_fs_loc) 469 ksft_exit_fail_msg("cannot create temp folder\n"); 470 471 return true; 472 } 473 474 static void cleanup_thp_fs(const char *thp_fs_loc, bool created_tmp) 475 { 476 int status; 477 478 if (!created_tmp) 479 return; 480 481 status = rmdir(thp_fs_loc); 482 if (status) 483 ksft_exit_fail_msg("cannot remove tmp dir: %s\n", 484 strerror(errno)); 485 } 486 487 static int create_pagecache_thp_and_fd(const char *testfile, size_t fd_size, 488 int *fd, char **addr) 489 { 490 size_t i; 491 unsigned char buf[1024]; 492 493 srand(time(NULL)); 494 495 *fd = open(testfile, O_CREAT | O_RDWR, 0664); 496 if (*fd == -1) 497 ksft_exit_fail_msg("Failed to create a file at %s\n", testfile); 498 499 assert(fd_size % sizeof(buf) == 0); 500 for (i = 0; i < sizeof(buf); i++) 501 buf[i] = (unsigned char)i; 502 for (i = 0; i < fd_size; i += sizeof(buf)) { 503 if (write(*fd, buf, sizeof(buf)) != sizeof(buf)) { 504 ksft_perror("write testfile"); 505 close(*fd); 506 goto err_out_unlink; 507 } 508 } 509 close(*fd); 510 sync(); 511 *fd = open("/proc/sys/vm/drop_caches", O_WRONLY); 512 if (*fd == -1) { 513 ksft_perror("open drop_caches"); 514 goto err_out_unlink; 515 } 516 if (write(*fd, "3", 1) != 1) { 517 ksft_perror("write to drop_caches"); 518 goto err_out_close; 519 } 520 close(*fd); 521 522 *fd = open(testfile, O_RDWR); 523 if (*fd == -1) { 524 ksft_perror("Failed to open testfile\n"); 525 goto err_out_unlink; 526 } 527 528 *addr = mmap(NULL, fd_size, PROT_READ|PROT_WRITE, MAP_SHARED, *fd, 0); 529 if (*addr == (char *)-1) { 530 ksft_perror("cannot mmap"); 531 goto err_out_close; 532 } 533 madvise(*addr, fd_size, MADV_HUGEPAGE); 534 535 force_read_pages(*addr, fd_size / pmd_pagesize, pmd_pagesize); 536 537 if (!check_huge_file(*addr, fd_size, fd_size / pmd_pagesize, pmd_pagesize)) { 538 ksft_print_msg("No large pagecache folio generated, please provide a filesystem supporting large folio\n"); 539 munmap(*addr, fd_size); 540 close(*fd); 541 unlink(testfile); 542 ksft_test_result_skip("Pagecache folio split skipped\n"); 543 return -2; 544 } 545 return 0; 546 err_out_close: 547 close(*fd); 548 err_out_unlink: 549 unlink(testfile); 550 ksft_exit_fail_msg("Failed to create large pagecache folios\n"); 551 return -1; 552 } 553 554 static void split_thp_in_pagecache_to_order_at(size_t fd_size, 555 const char *fs_loc, int order, int offset) 556 { 557 int fd; 558 char *split_addr; 559 char *addr; 560 size_t i; 561 char testfile[INPUT_MAX]; 562 int err = 0; 563 564 err = snprintf(testfile, INPUT_MAX, "%s/test", fs_loc); 565 566 if (err < 0) 567 ksft_exit_fail_msg("cannot generate right test file name\n"); 568 569 err = create_pagecache_thp_and_fd(testfile, fd_size, &fd, &addr); 570 if (err) 571 return; 572 573 err = 0; 574 575 memset(expected_orders, 0, sizeof(int) * (pmd_order + 1)); 576 /* 577 * use [split_addr, split_addr + pagesize) range to split THPs, since 578 * the debugfs function always split a range with pagesize step and 579 * providing a full [addr, addr + fd_size) range can trigger multiple 580 * splits, complicating after-split result checking. 581 */ 582 if (offset == -1) { 583 for (split_addr = addr; split_addr < addr + fd_size; split_addr += pmd_pagesize) 584 write_debugfs(PID_FMT, getpid(), (uint64_t)split_addr, 585 (uint64_t)split_addr + pagesize, order); 586 587 expected_orders[order] = fd_size / (pagesize << order); 588 } else { 589 int times = fd_size / pmd_pagesize; 590 591 for (split_addr = addr; split_addr < addr + fd_size; split_addr += pmd_pagesize) 592 write_debugfs(PID_FMT_OFFSET, getpid(), (uint64_t)split_addr, 593 (uint64_t)split_addr + pagesize, order, offset); 594 595 for (i = order + 1; i < pmd_order; i++) 596 expected_orders[i] = times; 597 expected_orders[order] = 2 * times; 598 } 599 600 for (i = 0; i < fd_size; i++) 601 if (*(addr + i) != (char)i) { 602 ksft_print_msg("%lu byte corrupted in the file\n", i); 603 err = EXIT_FAILURE; 604 goto out; 605 } 606 607 if (check_after_split_folio_orders(addr, fd_size, pagemap_fd, 608 kpageflags_fd, expected_orders, 609 (pmd_order + 1))) { 610 ksft_print_msg("Unexpected THP split\n"); 611 err = 1; 612 goto out; 613 } 614 615 if (!check_huge_file(addr, fd_size, 0, pmd_pagesize)) { 616 ksft_print_msg("Still FilePmdMapped not split\n"); 617 err = EXIT_FAILURE; 618 goto out; 619 } 620 621 out: 622 munmap(addr, fd_size); 623 close(fd); 624 unlink(testfile); 625 if (offset == -1) { 626 if (err) 627 ksft_exit_fail_msg("Split PMD-mapped pagecache folio to order %d failed\n", order); 628 ksft_test_result_pass("Split PMD-mapped pagecache folio to order %d passed\n", order); 629 } else { 630 if (err) 631 ksft_exit_fail_msg("Split PMD-mapped pagecache folio to order %d at in-folio offset %d failed\n", order, offset); 632 ksft_test_result_pass("Split PMD-mapped pagecache folio to order %d at in-folio offset %d passed\n", order, offset); 633 } 634 } 635 636 int main(int argc, char **argv) 637 { 638 int i; 639 size_t fd_size; 640 char *optional_xfs_path = NULL; 641 char fs_loc_template[] = "/tmp/thp_fs_XXXXXX"; 642 const char *fs_loc; 643 bool created_tmp; 644 int offset; 645 unsigned int nr_pages; 646 unsigned int tests; 647 648 ksft_print_header(); 649 650 if (geteuid() != 0) { 651 ksft_print_msg("Please run the benchmark as root\n"); 652 ksft_finished(); 653 } 654 655 if (!thp_is_enabled()) 656 ksft_exit_skip("Transparent Hugepages not available\n"); 657 658 if (argc > 1) 659 optional_xfs_path = argv[1]; 660 661 pagesize = getpagesize(); 662 pageshift = ffs(pagesize) - 1; 663 pmd_pagesize = read_pmd_pagesize(); 664 if (!pmd_pagesize) 665 ksft_exit_fail_msg("Reading PMD pagesize failed\n"); 666 667 nr_pages = pmd_pagesize / pagesize; 668 pmd_order = sz2ord(pmd_pagesize, pagesize); 669 670 expected_orders = (int *)malloc(sizeof(int) * (pmd_order + 1)); 671 if (!expected_orders) 672 ksft_exit_fail_msg("Fail to allocate memory: %s\n", strerror(errno)); 673 674 tests = 2 + (pmd_order - 1) + (2 * pmd_order) + (pmd_order - 1) * 4 + 2; 675 ksft_set_plan(tests); 676 677 pagemap_fd = open(pagemap_proc, O_RDONLY); 678 if (pagemap_fd == -1) 679 ksft_exit_fail_msg("read pagemap: %s\n", strerror(errno)); 680 681 kpageflags_fd = open(kpageflags_proc, O_RDONLY); 682 if (kpageflags_fd == -1) 683 ksft_exit_fail_msg("read kpageflags: %s\n", strerror(errno)); 684 685 fd_size = 2 * pmd_pagesize; 686 687 split_pmd_zero_pages(); 688 689 for (i = 0; i < pmd_order; i++) 690 if (i != 1) 691 split_pmd_thp_to_order(i); 692 693 split_pte_mapped_thp(); 694 for (i = 0; i < pmd_order; i++) 695 split_file_backed_thp(i); 696 697 created_tmp = prepare_thp_fs(optional_xfs_path, fs_loc_template, 698 &fs_loc); 699 for (i = pmd_order - 1; i >= 0; i--) 700 split_thp_in_pagecache_to_order_at(fd_size, fs_loc, i, -1); 701 702 for (i = 0; i < pmd_order; i++) 703 for (offset = 0; 704 offset < nr_pages; 705 offset += MAX(nr_pages / 4, 1 << i)) 706 split_thp_in_pagecache_to_order_at(fd_size, fs_loc, i, offset); 707 cleanup_thp_fs(fs_loc, created_tmp); 708 709 close(pagemap_fd); 710 close(kpageflags_fd); 711 free(expected_orders); 712 713 ksft_finished(); 714 715 return 0; 716 } 717