1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * A test of splitting PMD THPs and PTE-mapped THPs from a specified virtual 4 * address range in a process via <debugfs>/split_huge_pages interface. 5 */ 6 7 #define _GNU_SOURCE 8 #include <assert.h> 9 #include <stdio.h> 10 #include <stdlib.h> 11 #include <stdarg.h> 12 #include <unistd.h> 13 #include <inttypes.h> 14 #include <string.h> 15 #include <fcntl.h> 16 #include <sys/mman.h> 17 #include <sys/mount.h> 18 #include <sys/param.h> 19 #include <malloc.h> 20 #include <stdbool.h> 21 #include <time.h> 22 #include "vm_util.h" 23 #include "kselftest.h" 24 25 uint64_t pagesize; 26 unsigned int pageshift; 27 uint64_t pmd_pagesize; 28 unsigned int pmd_order; 29 int *expected_orders; 30 31 #define SPLIT_DEBUGFS "/sys/kernel/debug/split_huge_pages" 32 #define SMAP_PATH "/proc/self/smaps" 33 #define INPUT_MAX 80 34 35 #define PID_FMT "%d,0x%lx,0x%lx,%d" 36 #define PID_FMT_OFFSET "%d,0x%lx,0x%lx,%d,%d" 37 #define PATH_FMT "%s,0x%lx,0x%lx,%d" 38 39 const char *pagemap_proc = "/proc/self/pagemap"; 40 const char *kpageflags_proc = "/proc/kpageflags"; 41 int pagemap_fd; 42 int kpageflags_fd; 43 44 static bool is_backed_by_folio(char *vaddr, int order, int pagemap_fd, 45 int kpageflags_fd) 46 { 47 const uint64_t folio_head_flags = KPF_THP | KPF_COMPOUND_HEAD; 48 const uint64_t folio_tail_flags = KPF_THP | KPF_COMPOUND_TAIL; 49 const unsigned long nr_pages = 1UL << order; 50 unsigned long pfn_head; 51 uint64_t pfn_flags; 52 unsigned long pfn; 53 unsigned long i; 54 55 pfn = pagemap_get_pfn(pagemap_fd, vaddr); 56 57 /* non present page */ 58 if (pfn == -1UL) 59 return false; 60 61 if (pageflags_get(pfn, kpageflags_fd, &pfn_flags)) 62 goto fail; 63 64 /* check for order-0 pages */ 65 if (!order) { 66 if (pfn_flags & (folio_head_flags | folio_tail_flags)) 67 return false; 68 return true; 69 } 70 71 /* non THP folio */ 72 if (!(pfn_flags & KPF_THP)) 73 return false; 74 75 pfn_head = pfn & ~(nr_pages - 1); 76 77 if (pageflags_get(pfn_head, kpageflags_fd, &pfn_flags)) 78 goto fail; 79 80 /* head PFN has no compound_head flag set */ 81 if ((pfn_flags & folio_head_flags) != folio_head_flags) 82 return false; 83 84 /* check all tail PFN flags */ 85 for (i = 1; i < nr_pages; i++) { 86 if (pageflags_get(pfn_head + i, kpageflags_fd, &pfn_flags)) 87 goto fail; 88 if ((pfn_flags & folio_tail_flags) != folio_tail_flags) 89 return false; 90 } 91 92 /* 93 * check the PFN after this folio, but if its flags cannot be obtained, 94 * assume this folio has the expected order 95 */ 96 if (pageflags_get(pfn_head + nr_pages, kpageflags_fd, &pfn_flags)) 97 return true; 98 99 /* If we find another tail page, then the folio is larger. */ 100 return (pfn_flags & folio_tail_flags) != folio_tail_flags; 101 fail: 102 ksft_exit_fail_msg("Failed to get folio info\n"); 103 } 104 105 static int check_after_split_folio_orders(char *vaddr_start, size_t len, 106 int pagemap_fd, int kpageflags_fd, int orders[], int nr_orders) 107 { 108 int *vaddr_orders; 109 int status; 110 int i; 111 112 vaddr_orders = (int *)malloc(sizeof(int) * nr_orders); 113 114 if (!vaddr_orders) 115 ksft_exit_fail_msg("Cannot allocate memory for vaddr_orders"); 116 117 memset(vaddr_orders, 0, sizeof(int) * nr_orders); 118 status = gather_folio_orders(vaddr_start, len, pagemap_fd, 119 kpageflags_fd, vaddr_orders, nr_orders); 120 if (status) 121 ksft_exit_fail_msg("gather folio info failed\n"); 122 123 for (i = 0; i < nr_orders; i++) 124 if (vaddr_orders[i] != orders[i]) { 125 ksft_print_msg("order %d: expected: %d got %d\n", i, 126 orders[i], vaddr_orders[i]); 127 status = -1; 128 } 129 130 free(vaddr_orders); 131 return status; 132 } 133 134 static void write_debugfs(const char *fmt, ...) 135 { 136 char input[INPUT_MAX]; 137 int ret; 138 va_list argp; 139 140 va_start(argp, fmt); 141 ret = vsnprintf(input, INPUT_MAX, fmt, argp); 142 va_end(argp); 143 144 if (ret >= INPUT_MAX) 145 ksft_exit_fail_msg("%s: Debugfs input is too long\n", __func__); 146 147 ret = write_file(SPLIT_DEBUGFS, input, ret + 1); 148 if (ret) 149 ksft_exit_fail_msg("write_file(%s): %s\n", SPLIT_DEBUGFS, 150 strerror(-ret)); 151 } 152 153 static char *allocate_zero_filled_hugepage(size_t len) 154 { 155 char *result; 156 size_t i; 157 158 result = memalign(pmd_pagesize, len); 159 if (!result) { 160 printf("Fail to allocate memory\n"); 161 exit(EXIT_FAILURE); 162 } 163 164 madvise(result, len, MADV_HUGEPAGE); 165 166 for (i = 0; i < len; i++) 167 result[i] = (char)0; 168 169 return result; 170 } 171 172 static void verify_rss_anon_split_huge_page_all_zeroes(char *one_page, int nr_hpages, size_t len) 173 { 174 unsigned long rss_anon_before, rss_anon_after; 175 size_t i; 176 177 if (!check_huge_anon(one_page, nr_hpages * pmd_pagesize, nr_hpages, pmd_pagesize)) 178 ksft_exit_fail_msg("No THP is allocated\n"); 179 180 rss_anon_before = rss_anon(); 181 if (!rss_anon_before) 182 ksft_exit_fail_msg("No RssAnon is allocated before split\n"); 183 184 /* split all THPs */ 185 write_debugfs(PID_FMT, getpid(), (uint64_t)one_page, 186 (uint64_t)one_page + len, 0); 187 188 for (i = 0; i < len; i++) 189 if (one_page[i] != (char)0) 190 ksft_exit_fail_msg("%ld byte corrupted\n", i); 191 192 if (!check_huge_anon(one_page, nr_hpages * pmd_pagesize, 0, pmd_pagesize)) 193 ksft_exit_fail_msg("Still AnonHugePages not split\n"); 194 195 rss_anon_after = rss_anon(); 196 if (rss_anon_after >= rss_anon_before) 197 ksft_exit_fail_msg("Incorrect RssAnon value. Before: %ld After: %ld\n", 198 rss_anon_before, rss_anon_after); 199 } 200 201 static void split_pmd_zero_pages(void) 202 { 203 char *one_page; 204 int nr_hpages = 4; 205 size_t len = nr_hpages * pmd_pagesize; 206 207 one_page = allocate_zero_filled_hugepage(len); 208 verify_rss_anon_split_huge_page_all_zeroes(one_page, nr_hpages, len); 209 ksft_test_result_pass("Split zero filled huge pages successful\n"); 210 free(one_page); 211 } 212 213 static void split_pmd_thp_to_order(int order) 214 { 215 char *one_page; 216 size_t len = 4 * pmd_pagesize; 217 size_t i; 218 219 one_page = memalign(pmd_pagesize, len); 220 if (!one_page) 221 ksft_exit_fail_msg("Fail to allocate memory: %s\n", strerror(errno)); 222 223 madvise(one_page, len, MADV_HUGEPAGE); 224 225 for (i = 0; i < len; i++) 226 one_page[i] = (char)i; 227 228 if (!check_huge_anon(one_page, 4 * pmd_pagesize, 4, pmd_pagesize)) 229 ksft_exit_fail_msg("No THP is allocated\n"); 230 231 /* split all THPs */ 232 write_debugfs(PID_FMT, getpid(), (uint64_t)one_page, 233 (uint64_t)one_page + len, order); 234 235 for (i = 0; i < len; i++) 236 if (one_page[i] != (char)i) 237 ksft_exit_fail_msg("%ld byte corrupted\n", i); 238 239 memset(expected_orders, 0, sizeof(int) * (pmd_order + 1)); 240 expected_orders[order] = 4 << (pmd_order - order); 241 242 if (check_after_split_folio_orders(one_page, len, pagemap_fd, 243 kpageflags_fd, expected_orders, 244 (pmd_order + 1))) 245 ksft_exit_fail_msg("Unexpected THP split\n"); 246 247 if (!check_huge_anon(one_page, 4 * pmd_pagesize, 0, pmd_pagesize)) 248 ksft_exit_fail_msg("Still AnonHugePages not split\n"); 249 250 ksft_test_result_pass("Split huge pages to order %d successful\n", order); 251 free(one_page); 252 } 253 254 static void split_pte_mapped_thp(void) 255 { 256 const size_t nr_thps = 4; 257 const size_t thp_area_size = nr_thps * pmd_pagesize; 258 const size_t page_area_size = nr_thps * pagesize; 259 char *thp_area, *tmp, *page_area = MAP_FAILED; 260 size_t i; 261 262 thp_area = mmap((void *)(1UL << 30), thp_area_size, PROT_READ | PROT_WRITE, 263 MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); 264 if (thp_area == MAP_FAILED) { 265 ksft_test_result_fail("Fail to allocate memory: %s\n", strerror(errno)); 266 return; 267 } 268 269 madvise(thp_area, thp_area_size, MADV_HUGEPAGE); 270 271 for (i = 0; i < thp_area_size; i++) 272 thp_area[i] = (char)i; 273 274 if (!check_huge_anon(thp_area, nr_thps * pmd_pagesize, nr_thps, pmd_pagesize)) { 275 ksft_test_result_skip("Not all THPs allocated\n"); 276 goto out; 277 } 278 279 /* 280 * To challenge spitting code, we will mremap a single page of each 281 * THP (page[i] of thp[i]) in the thp_area into page_area. This will 282 * replace the PMD mappings in the thp_area by PTE mappings first, 283 * but leaving the THP unsplit, to then create a page-sized hole in 284 * the thp_area. 285 * We will then manually trigger splitting of all THPs through the 286 * single mremap'ed pages of each THP in the page_area. 287 */ 288 page_area = mmap(NULL, page_area_size, PROT_READ | PROT_WRITE, 289 MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); 290 if (page_area == MAP_FAILED) { 291 ksft_test_result_fail("Fail to allocate memory: %s\n", strerror(errno)); 292 goto out; 293 } 294 295 for (i = 0; i < nr_thps; i++) { 296 tmp = mremap(thp_area + pmd_pagesize * i + pagesize * i, 297 pagesize, pagesize, MREMAP_MAYMOVE|MREMAP_FIXED, 298 page_area + pagesize * i); 299 if (tmp != MAP_FAILED) 300 continue; 301 ksft_test_result_fail("mremap failed: %s\n", strerror(errno)); 302 goto out; 303 } 304 305 /* 306 * Verify that our THPs were not split yet. Note that 307 * check_huge_anon() cannot be used as it checks for PMD mappings. 308 */ 309 for (i = 0; i < nr_thps; i++) { 310 if (is_backed_by_folio(page_area + i * pagesize, pmd_order, 311 pagemap_fd, kpageflags_fd)) 312 continue; 313 ksft_test_result_fail("THP %zu missing after mremap\n", i); 314 goto out; 315 } 316 317 /* Split all THPs through the remapped pages. */ 318 write_debugfs(PID_FMT, getpid(), (uint64_t)page_area, 319 (uint64_t)page_area + page_area_size, 0); 320 321 /* Corruption during mremap or split? */ 322 for (i = 0; i < page_area_size; i++) { 323 if (page_area[i] == (char)i) 324 continue; 325 ksft_test_result_fail("%zu byte corrupted\n", i); 326 goto out; 327 } 328 329 /* Split failed? */ 330 for (i = 0; i < nr_thps; i++) { 331 if (is_backed_by_folio(page_area + i * pagesize, 0, 332 pagemap_fd, kpageflags_fd)) 333 continue; 334 ksft_test_result_fail("THP %zu not split\n", i); 335 } 336 337 ksft_test_result_pass("Split PTE-mapped huge pages successful\n"); 338 out: 339 munmap(thp_area, thp_area_size); 340 if (page_area != MAP_FAILED) 341 munmap(page_area, page_area_size); 342 } 343 344 static void split_file_backed_thp(int order) 345 { 346 int status; 347 int fd; 348 char tmpfs_template[] = "/tmp/thp_split_XXXXXX"; 349 const char *tmpfs_loc = mkdtemp(tmpfs_template); 350 char testfile[INPUT_MAX]; 351 unsigned long size = 2 * pmd_pagesize; 352 char opts[64]; 353 ssize_t num_written, num_read; 354 char *file_buf1 = NULL, *file_buf2 = NULL; 355 uint64_t pgoff_start = 0, pgoff_end = 1024; 356 int i; 357 358 ksft_print_msg("Please enable pr_debug in split_huge_pages_in_file() for more info.\n"); 359 360 if (!tmpfs_loc) 361 ksft_exit_fail_msg("mkdtemp failed\n"); 362 363 file_buf1 = (char *)malloc(pmd_pagesize); 364 file_buf2 = (char *)malloc(pmd_pagesize); 365 366 if (!file_buf1 || !file_buf2) { 367 ksft_print_msg("cannot allocate file buffers\n"); 368 goto out; 369 } 370 371 for (i = 0; i < pmd_pagesize; i++) 372 file_buf1[i] = (char)i; 373 memset(file_buf2, 0, pmd_pagesize); 374 375 snprintf(opts, sizeof(opts), "huge=always,size=%lu", size); 376 status = mount("tmpfs", tmpfs_loc, "tmpfs", 0, opts); 377 378 if (status) { 379 ksft_print_msg("Unable to create a tmpfs for testing\n"); 380 goto out; 381 } 382 383 status = snprintf(testfile, INPUT_MAX, "%s/thp_file", tmpfs_loc); 384 if (status >= INPUT_MAX) { 385 ksft_print_msg("Fail to create file-backed THP split testing file\n"); 386 goto cleanup; 387 } 388 389 fd = open(testfile, O_CREAT|O_RDWR, 0664); 390 if (fd == -1) { 391 ksft_perror("Cannot open testing file"); 392 goto cleanup; 393 } 394 395 /* write pmd size data to the file, so a file-backed THP can be allocated */ 396 num_written = write(fd, file_buf1, pmd_pagesize); 397 398 if (num_written == -1 || num_written != pmd_pagesize) { 399 ksft_perror("Failed to write data to testing file"); 400 goto close_file; 401 } 402 403 /* split the file-backed THP */ 404 write_debugfs(PATH_FMT, testfile, pgoff_start, pgoff_end, order); 405 406 /* check file content after split */ 407 status = lseek(fd, 0, SEEK_SET); 408 if (status == -1) { 409 ksft_perror("Cannot lseek file"); 410 goto close_file; 411 } 412 413 num_read = read(fd, file_buf2, num_written); 414 if (num_read == -1 || num_read != num_written) { 415 ksft_perror("Cannot read file content back"); 416 goto close_file; 417 } 418 419 if (strncmp(file_buf1, file_buf2, pmd_pagesize) != 0) { 420 ksft_print_msg("File content changed\n"); 421 goto close_file; 422 } 423 424 close(fd); 425 status = unlink(testfile); 426 if (status) { 427 ksft_perror("Cannot remove testing file"); 428 goto cleanup; 429 } 430 431 status = umount(tmpfs_loc); 432 if (status) { 433 ksft_print_msg("Unable to umount %s\n", tmpfs_loc); 434 goto out; 435 } 436 437 free(file_buf1); 438 free(file_buf2); 439 440 status = rmdir(tmpfs_loc); 441 if (status) 442 ksft_exit_fail_msg("cannot remove tmp dir: %s\n", strerror(errno)); 443 444 ksft_print_msg("Please check dmesg for more information\n"); 445 ksft_test_result_pass("File-backed THP split to order %d test done\n", order); 446 return; 447 448 close_file: 449 close(fd); 450 cleanup: 451 umount(tmpfs_loc); 452 out: 453 free(file_buf1); 454 free(file_buf2); 455 rmdir(tmpfs_loc); 456 ksft_exit_fail_msg("Error occurred\n"); 457 } 458 459 static bool prepare_thp_fs(const char *xfs_path, char *thp_fs_template, 460 const char **thp_fs_loc) 461 { 462 if (xfs_path) { 463 *thp_fs_loc = xfs_path; 464 return false; 465 } 466 467 *thp_fs_loc = mkdtemp(thp_fs_template); 468 469 if (!*thp_fs_loc) 470 ksft_exit_fail_msg("cannot create temp folder\n"); 471 472 return true; 473 } 474 475 static void cleanup_thp_fs(const char *thp_fs_loc, bool created_tmp) 476 { 477 int status; 478 479 if (!created_tmp) 480 return; 481 482 status = rmdir(thp_fs_loc); 483 if (status) 484 ksft_exit_fail_msg("cannot remove tmp dir: %s\n", 485 strerror(errno)); 486 } 487 488 static int create_pagecache_thp_and_fd(const char *testfile, size_t fd_size, 489 int *fd, char **addr) 490 { 491 size_t i; 492 unsigned char buf[1024]; 493 494 srand(time(NULL)); 495 496 *fd = open(testfile, O_CREAT | O_RDWR, 0664); 497 if (*fd == -1) 498 ksft_exit_fail_msg("Failed to create a file at %s\n", testfile); 499 500 assert(fd_size % sizeof(buf) == 0); 501 for (i = 0; i < sizeof(buf); i++) 502 buf[i] = (unsigned char)i; 503 for (i = 0; i < fd_size; i += sizeof(buf)) { 504 if (write(*fd, buf, sizeof(buf)) != sizeof(buf)) { 505 ksft_perror("write testfile"); 506 close(*fd); 507 goto err_out_unlink; 508 } 509 } 510 close(*fd); 511 sync(); 512 *fd = open("/proc/sys/vm/drop_caches", O_WRONLY); 513 if (*fd == -1) { 514 ksft_perror("open drop_caches"); 515 goto err_out_unlink; 516 } 517 if (write(*fd, "3", 1) != 1) { 518 ksft_perror("write to drop_caches"); 519 goto err_out_close; 520 } 521 close(*fd); 522 523 *fd = open(testfile, O_RDWR); 524 if (*fd == -1) { 525 ksft_perror("Failed to open testfile\n"); 526 goto err_out_unlink; 527 } 528 529 *addr = mmap(NULL, fd_size, PROT_READ|PROT_WRITE, MAP_SHARED, *fd, 0); 530 if (*addr == (char *)-1) { 531 ksft_perror("cannot mmap"); 532 goto err_out_close; 533 } 534 madvise(*addr, fd_size, MADV_HUGEPAGE); 535 536 force_read_pages(*addr, fd_size / pmd_pagesize, pmd_pagesize); 537 538 if (!check_huge_file(*addr, fd_size, fd_size / pmd_pagesize, pmd_pagesize)) { 539 ksft_print_msg("No large pagecache folio generated, please provide a filesystem supporting large folio\n"); 540 munmap(*addr, fd_size); 541 close(*fd); 542 unlink(testfile); 543 ksft_test_result_skip("Pagecache folio split skipped\n"); 544 return -2; 545 } 546 return 0; 547 err_out_close: 548 close(*fd); 549 err_out_unlink: 550 unlink(testfile); 551 ksft_exit_fail_msg("Failed to create large pagecache folios\n"); 552 } 553 554 static void split_thp_in_pagecache_to_order_at(size_t fd_size, 555 const char *fs_loc, int order, int offset) 556 { 557 int fd; 558 char *split_addr; 559 char *addr; 560 size_t i; 561 char testfile[INPUT_MAX]; 562 int err = 0; 563 564 err = snprintf(testfile, INPUT_MAX, "%s/test", fs_loc); 565 566 if (err < 0) 567 ksft_exit_fail_msg("cannot generate right test file name\n"); 568 569 err = create_pagecache_thp_and_fd(testfile, fd_size, &fd, &addr); 570 if (err) 571 return; 572 573 err = 0; 574 575 memset(expected_orders, 0, sizeof(int) * (pmd_order + 1)); 576 /* 577 * use [split_addr, split_addr + pagesize) range to split THPs, since 578 * the debugfs function always split a range with pagesize step and 579 * providing a full [addr, addr + fd_size) range can trigger multiple 580 * splits, complicating after-split result checking. 581 */ 582 if (offset == -1) { 583 for (split_addr = addr; split_addr < addr + fd_size; split_addr += pmd_pagesize) 584 write_debugfs(PID_FMT, getpid(), (uint64_t)split_addr, 585 (uint64_t)split_addr + pagesize, order); 586 587 expected_orders[order] = fd_size / (pagesize << order); 588 } else { 589 int times = fd_size / pmd_pagesize; 590 591 for (split_addr = addr; split_addr < addr + fd_size; split_addr += pmd_pagesize) 592 write_debugfs(PID_FMT_OFFSET, getpid(), (uint64_t)split_addr, 593 (uint64_t)split_addr + pagesize, order, offset); 594 595 for (i = order + 1; i < pmd_order; i++) 596 expected_orders[i] = times; 597 expected_orders[order] = 2 * times; 598 } 599 600 for (i = 0; i < fd_size; i++) 601 if (*(addr + i) != (char)i) { 602 ksft_print_msg("%lu byte corrupted in the file\n", i); 603 err = EXIT_FAILURE; 604 goto out; 605 } 606 607 if (check_after_split_folio_orders(addr, fd_size, pagemap_fd, 608 kpageflags_fd, expected_orders, 609 (pmd_order + 1))) { 610 ksft_print_msg("Unexpected THP split\n"); 611 err = 1; 612 goto out; 613 } 614 615 if (!check_huge_file(addr, fd_size, 0, pmd_pagesize)) { 616 ksft_print_msg("Still FilePmdMapped not split\n"); 617 err = EXIT_FAILURE; 618 goto out; 619 } 620 621 out: 622 munmap(addr, fd_size); 623 close(fd); 624 unlink(testfile); 625 if (offset == -1) { 626 if (err) 627 ksft_exit_fail_msg("Split PMD-mapped pagecache folio to order %d failed\n", order); 628 ksft_test_result_pass("Split PMD-mapped pagecache folio to order %d passed\n", order); 629 } else { 630 if (err) 631 ksft_exit_fail_msg("Split PMD-mapped pagecache folio to order %d at in-folio offset %d failed\n", order, offset); 632 ksft_test_result_pass("Split PMD-mapped pagecache folio to order %d at in-folio offset %d passed\n", order, offset); 633 } 634 } 635 636 int main(int argc, char **argv) 637 { 638 int i; 639 size_t fd_size; 640 char *optional_xfs_path = NULL; 641 char fs_loc_template[] = "/tmp/thp_fs_XXXXXX"; 642 const char *fs_loc; 643 bool created_tmp; 644 int offset; 645 unsigned int nr_pages; 646 unsigned int tests; 647 648 ksft_print_header(); 649 650 if (geteuid() != 0) { 651 ksft_print_msg("Please run the benchmark as root\n"); 652 ksft_finished(); 653 } 654 655 if (!thp_is_enabled()) 656 ksft_exit_skip("Transparent Hugepages not available\n"); 657 658 if (argc > 1) 659 optional_xfs_path = argv[1]; 660 661 pagesize = getpagesize(); 662 pageshift = ffs(pagesize) - 1; 663 pmd_pagesize = read_pmd_pagesize(); 664 if (!pmd_pagesize) 665 ksft_exit_fail_msg("Reading PMD pagesize failed\n"); 666 667 nr_pages = pmd_pagesize / pagesize; 668 pmd_order = sz2ord(pmd_pagesize, pagesize); 669 670 expected_orders = (int *)malloc(sizeof(int) * (pmd_order + 1)); 671 if (!expected_orders) 672 ksft_exit_fail_msg("Fail to allocate memory: %s\n", strerror(errno)); 673 674 tests = 2 + (pmd_order - 1) + (2 * pmd_order) + (pmd_order - 1) * 4 + 2; 675 ksft_set_plan(tests); 676 677 pagemap_fd = open(pagemap_proc, O_RDONLY); 678 if (pagemap_fd == -1) 679 ksft_exit_fail_msg("read pagemap: %s\n", strerror(errno)); 680 681 kpageflags_fd = open(kpageflags_proc, O_RDONLY); 682 if (kpageflags_fd == -1) 683 ksft_exit_fail_msg("read kpageflags: %s\n", strerror(errno)); 684 685 fd_size = 2 * pmd_pagesize; 686 687 split_pmd_zero_pages(); 688 689 for (i = 0; i < pmd_order; i++) 690 if (i != 1) 691 split_pmd_thp_to_order(i); 692 693 split_pte_mapped_thp(); 694 for (i = 0; i < pmd_order; i++) 695 split_file_backed_thp(i); 696 697 created_tmp = prepare_thp_fs(optional_xfs_path, fs_loc_template, 698 &fs_loc); 699 for (i = pmd_order - 1; i >= 0; i--) 700 split_thp_in_pagecache_to_order_at(fd_size, fs_loc, i, -1); 701 702 for (i = 0; i < pmd_order; i++) 703 for (offset = 0; 704 offset < nr_pages; 705 offset += MAX(nr_pages / 4, 1 << i)) 706 split_thp_in_pagecache_to_order_at(fd_size, fs_loc, i, offset); 707 cleanup_thp_fs(fs_loc, created_tmp); 708 709 close(pagemap_fd); 710 close(kpageflags_fd); 711 free(expected_orders); 712 713 ksft_finished(); 714 } 715