1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * A test of splitting PMD THPs and PTE-mapped THPs from a specified virtual 4 * address range in a process via <debugfs>/split_huge_pages interface. 5 */ 6 7 #define _GNU_SOURCE 8 #include <assert.h> 9 #include <stdio.h> 10 #include <stdlib.h> 11 #include <stdarg.h> 12 #include <unistd.h> 13 #include <inttypes.h> 14 #include <string.h> 15 #include <fcntl.h> 16 #include <sys/mman.h> 17 #include <sys/mount.h> 18 #include <sys/param.h> 19 #include <malloc.h> 20 #include <stdbool.h> 21 #include <time.h> 22 #include "vm_util.h" 23 #include "kselftest.h" 24 25 uint64_t pagesize; 26 unsigned int pageshift; 27 uint64_t pmd_pagesize; 28 unsigned int pmd_order; 29 int *expected_orders; 30 31 #define SPLIT_DEBUGFS "/sys/kernel/debug/split_huge_pages" 32 #define SMAP_PATH "/proc/self/smaps" 33 #define INPUT_MAX 80 34 35 #define PID_FMT "%d,0x%lx,0x%lx,%d" 36 #define PID_FMT_OFFSET "%d,0x%lx,0x%lx,%d,%d" 37 #define PATH_FMT "%s,0x%lx,0x%lx,%d" 38 39 const char *pagemap_proc = "/proc/self/pagemap"; 40 const char *kpageflags_proc = "/proc/kpageflags"; 41 int pagemap_fd; 42 int kpageflags_fd; 43 44 static int check_after_split_folio_orders(char *vaddr_start, size_t len, 45 int pagemap_fd, int kpageflags_fd, int orders[], int nr_orders) 46 { 47 int *vaddr_orders; 48 int status; 49 int i; 50 51 vaddr_orders = (int *)malloc(sizeof(int) * nr_orders); 52 53 if (!vaddr_orders) 54 ksft_exit_fail_msg("Cannot allocate memory for vaddr_orders"); 55 56 memset(vaddr_orders, 0, sizeof(int) * nr_orders); 57 status = gather_folio_orders(vaddr_start, len, pagemap_fd, 58 kpageflags_fd, vaddr_orders, nr_orders); 59 if (status) 60 ksft_exit_fail_msg("gather folio info failed\n"); 61 62 for (i = 0; i < nr_orders; i++) 63 if (vaddr_orders[i] != orders[i]) { 64 ksft_print_msg("order %d: expected: %d got %d\n", i, 65 orders[i], vaddr_orders[i]); 66 status = -1; 67 } 68 69 free(vaddr_orders); 70 return status; 71 } 72 73 static void write_debugfs(const char *fmt, ...) 74 { 75 char input[INPUT_MAX]; 76 int ret; 77 va_list argp; 78 79 va_start(argp, fmt); 80 ret = vsnprintf(input, INPUT_MAX, fmt, argp); 81 va_end(argp); 82 83 if (ret >= INPUT_MAX) 84 ksft_exit_fail_msg("%s: Debugfs input is too long\n", __func__); 85 86 ret = write_file(SPLIT_DEBUGFS, input, ret + 1); 87 if (ret) 88 ksft_exit_fail_msg("write_file(%s): %s\n", SPLIT_DEBUGFS, 89 strerror(-ret)); 90 } 91 92 static char *allocate_zero_filled_hugepage(size_t len) 93 { 94 char *result; 95 size_t i; 96 97 result = memalign(pmd_pagesize, len); 98 if (!result) { 99 printf("Fail to allocate memory\n"); 100 exit(EXIT_FAILURE); 101 } 102 103 madvise(result, len, MADV_HUGEPAGE); 104 105 for (i = 0; i < len; i++) 106 result[i] = (char)0; 107 108 return result; 109 } 110 111 static void disable_khugepaged(void *addr, size_t len) 112 { 113 /* Disables khugepaged from collapsing pages in range into THPs */ 114 if (!madvise(addr, len, MADV_NOHUGEPAGE)) 115 return; 116 117 ksft_exit_fail_perror("MADV_NOHUGEPAGE failed"); 118 } 119 120 static void verify_rss_anon_split_huge_page_all_zeroes(char *one_page, int nr_hpages, size_t len) 121 { 122 unsigned long rss_anon_before, rss_anon_after; 123 size_t i; 124 125 if (!check_huge_anon(one_page, nr_hpages * pmd_pagesize, nr_hpages, pmd_pagesize)) 126 ksft_exit_fail_msg("No THP is allocated\n"); 127 128 rss_anon_before = rss_anon(); 129 if (!rss_anon_before) 130 ksft_exit_fail_msg("No RssAnon is allocated before split\n"); 131 132 disable_khugepaged(one_page, len); 133 134 /* split all THPs */ 135 write_debugfs(PID_FMT, getpid(), (uint64_t)one_page, 136 (uint64_t)one_page + len, 0); 137 138 for (i = 0; i < len; i++) 139 if (one_page[i] != (char)0) 140 ksft_exit_fail_msg("%ld byte corrupted\n", i); 141 142 if (!check_huge_anon(one_page, nr_hpages * pmd_pagesize, 0, pmd_pagesize)) 143 ksft_exit_fail_msg("Still AnonHugePages not split\n"); 144 145 rss_anon_after = rss_anon(); 146 if (rss_anon_after >= rss_anon_before) 147 ksft_exit_fail_msg("Incorrect RssAnon value. Before: %ld After: %ld\n", 148 rss_anon_before, rss_anon_after); 149 } 150 151 static void split_pmd_zero_pages(void) 152 { 153 char *one_page; 154 int nr_hpages = 4; 155 size_t len = nr_hpages * pmd_pagesize; 156 157 one_page = allocate_zero_filled_hugepage(len); 158 verify_rss_anon_split_huge_page_all_zeroes(one_page, nr_hpages, len); 159 ksft_test_result_pass("Split zero filled huge pages successful\n"); 160 free(one_page); 161 } 162 163 static void split_pmd_thp_to_order(int order) 164 { 165 char *one_page; 166 size_t len = 4 * pmd_pagesize; 167 size_t i; 168 169 one_page = memalign(pmd_pagesize, len); 170 if (!one_page) 171 ksft_exit_fail_msg("Fail to allocate memory: %s\n", strerror(errno)); 172 173 madvise(one_page, len, MADV_HUGEPAGE); 174 175 for (i = 0; i < len; i++) 176 one_page[i] = (char)i; 177 178 if (!check_huge_anon(one_page, 4 * pmd_pagesize, 4, pmd_pagesize)) 179 ksft_exit_fail_msg("No THP is allocated\n"); 180 181 disable_khugepaged(one_page, len); 182 183 /* split all THPs */ 184 write_debugfs(PID_FMT, getpid(), (uint64_t)one_page, 185 (uint64_t)one_page + len, order); 186 187 for (i = 0; i < len; i++) 188 if (one_page[i] != (char)i) 189 ksft_exit_fail_msg("%ld byte corrupted\n", i); 190 191 memset(expected_orders, 0, sizeof(int) * (pmd_order + 1)); 192 expected_orders[order] = 4 << (pmd_order - order); 193 194 if (check_after_split_folio_orders(one_page, len, pagemap_fd, 195 kpageflags_fd, expected_orders, 196 (pmd_order + 1))) 197 ksft_exit_fail_msg("Unexpected THP split\n"); 198 199 if (!check_huge_anon(one_page, 4 * pmd_pagesize, 0, pmd_pagesize)) 200 ksft_exit_fail_msg("Still AnonHugePages not split\n"); 201 202 ksft_test_result_pass("Split huge pages to order %d successful\n", order); 203 free(one_page); 204 } 205 206 static void split_pte_mapped_thp(void) 207 { 208 const size_t nr_thps = 4; 209 const size_t thp_area_size = nr_thps * pmd_pagesize; 210 const size_t page_area_size = nr_thps * pagesize; 211 char *thp_area, *tmp, *page_area = MAP_FAILED; 212 size_t i; 213 214 thp_area = mmap((void *)(1UL << 30), thp_area_size, PROT_READ | PROT_WRITE, 215 MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); 216 if (thp_area == MAP_FAILED) { 217 ksft_test_result_fail("Fail to allocate memory: %s\n", strerror(errno)); 218 return; 219 } 220 221 madvise(thp_area, thp_area_size, MADV_HUGEPAGE); 222 223 for (i = 0; i < thp_area_size; i++) 224 thp_area[i] = (char)i; 225 226 if (!check_huge_anon(thp_area, nr_thps * pmd_pagesize, nr_thps, pmd_pagesize)) { 227 ksft_test_result_skip("Not all THPs allocated\n"); 228 goto out; 229 } 230 231 disable_khugepaged(thp_area, thp_area_size); 232 233 /* 234 * To challenge spitting code, we will mremap a single page of each 235 * THP (page[i] of thp[i]) in the thp_area into page_area. This will 236 * replace the PMD mappings in the thp_area by PTE mappings first, 237 * but leaving the THP unsplit, to then create a page-sized hole in 238 * the thp_area. 239 * We will then manually trigger splitting of all THPs through the 240 * single mremap'ed pages of each THP in the page_area. 241 */ 242 page_area = mmap(NULL, page_area_size, PROT_READ | PROT_WRITE, 243 MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); 244 if (page_area == MAP_FAILED) { 245 ksft_test_result_fail("Fail to allocate memory: %s\n", strerror(errno)); 246 goto out; 247 } 248 249 for (i = 0; i < nr_thps; i++) { 250 tmp = mremap(thp_area + pmd_pagesize * i + pagesize * i, 251 pagesize, pagesize, MREMAP_MAYMOVE|MREMAP_FIXED, 252 page_area + pagesize * i); 253 if (tmp != MAP_FAILED) 254 continue; 255 ksft_test_result_fail("mremap failed: %s\n", strerror(errno)); 256 goto out; 257 } 258 259 /* 260 * Verify that our THPs were not split yet. Note that 261 * check_huge_anon() cannot be used as it checks for PMD mappings. 262 */ 263 for (i = 0; i < nr_thps; i++) { 264 if (is_backed_by_folio(page_area + i * pagesize, pmd_order, 265 pagemap_fd, kpageflags_fd)) 266 continue; 267 ksft_test_result_fail("THP %zu missing after mremap\n", i); 268 goto out; 269 } 270 271 /* Split all THPs through the remapped pages. */ 272 write_debugfs(PID_FMT, getpid(), (uint64_t)page_area, 273 (uint64_t)page_area + page_area_size, 0); 274 275 /* Corruption during mremap or split? */ 276 for (i = 0; i < page_area_size; i++) { 277 if (page_area[i] == (char)i) 278 continue; 279 ksft_test_result_fail("%zu byte corrupted\n", i); 280 goto out; 281 } 282 283 /* Split failed? */ 284 for (i = 0; i < nr_thps; i++) { 285 if (is_backed_by_folio(page_area + i * pagesize, 0, 286 pagemap_fd, kpageflags_fd)) 287 continue; 288 ksft_test_result_fail("THP %zu not split\n", i); 289 } 290 291 ksft_test_result_pass("Split PTE-mapped huge pages successful\n"); 292 out: 293 munmap(thp_area, thp_area_size); 294 if (page_area != MAP_FAILED) 295 munmap(page_area, page_area_size); 296 } 297 298 static void split_file_backed_thp(int order) 299 { 300 int status; 301 int fd; 302 char tmpfs_template[] = "/tmp/thp_split_XXXXXX"; 303 const char *tmpfs_loc = mkdtemp(tmpfs_template); 304 char testfile[INPUT_MAX]; 305 unsigned long size = 2 * pmd_pagesize; 306 char opts[64]; 307 ssize_t num_written, num_read; 308 char *file_buf1 = NULL, *file_buf2 = NULL; 309 uint64_t pgoff_start = 0, pgoff_end = 1024; 310 int i; 311 312 ksft_print_msg("Please enable pr_debug in split_huge_pages_in_file() for more info.\n"); 313 314 if (!tmpfs_loc) 315 ksft_exit_fail_msg("mkdtemp failed\n"); 316 317 file_buf1 = (char *)malloc(pmd_pagesize); 318 file_buf2 = (char *)malloc(pmd_pagesize); 319 320 if (!file_buf1 || !file_buf2) { 321 ksft_print_msg("cannot allocate file buffers\n"); 322 goto out; 323 } 324 325 for (i = 0; i < pmd_pagesize; i++) 326 file_buf1[i] = (char)i; 327 memset(file_buf2, 0, pmd_pagesize); 328 329 snprintf(opts, sizeof(opts), "huge=always,size=%lu", size); 330 status = mount("tmpfs", tmpfs_loc, "tmpfs", 0, opts); 331 332 if (status) { 333 ksft_print_msg("Unable to create a tmpfs for testing\n"); 334 goto out; 335 } 336 337 status = snprintf(testfile, INPUT_MAX, "%s/thp_file", tmpfs_loc); 338 if (status >= INPUT_MAX) { 339 ksft_print_msg("Fail to create file-backed THP split testing file\n"); 340 goto cleanup; 341 } 342 343 fd = open(testfile, O_CREAT|O_RDWR, 0664); 344 if (fd == -1) { 345 ksft_perror("Cannot open testing file"); 346 goto cleanup; 347 } 348 349 /* write pmd size data to the file, so a file-backed THP can be allocated */ 350 num_written = write(fd, file_buf1, pmd_pagesize); 351 352 if (num_written == -1 || num_written != pmd_pagesize) { 353 ksft_perror("Failed to write data to testing file"); 354 goto close_file; 355 } 356 357 /* split the file-backed THP */ 358 write_debugfs(PATH_FMT, testfile, pgoff_start, pgoff_end, order); 359 360 /* check file content after split */ 361 status = lseek(fd, 0, SEEK_SET); 362 if (status == -1) { 363 ksft_perror("Cannot lseek file"); 364 goto close_file; 365 } 366 367 num_read = read(fd, file_buf2, num_written); 368 if (num_read == -1 || num_read != num_written) { 369 ksft_perror("Cannot read file content back"); 370 goto close_file; 371 } 372 373 if (strncmp(file_buf1, file_buf2, pmd_pagesize) != 0) { 374 ksft_print_msg("File content changed\n"); 375 goto close_file; 376 } 377 378 close(fd); 379 status = unlink(testfile); 380 if (status) { 381 ksft_perror("Cannot remove testing file"); 382 goto cleanup; 383 } 384 385 status = umount(tmpfs_loc); 386 if (status) { 387 ksft_print_msg("Unable to umount %s\n", tmpfs_loc); 388 goto out; 389 } 390 391 free(file_buf1); 392 free(file_buf2); 393 394 status = rmdir(tmpfs_loc); 395 if (status) 396 ksft_exit_fail_msg("cannot remove tmp dir: %s\n", strerror(errno)); 397 398 ksft_print_msg("Please check dmesg for more information\n"); 399 ksft_test_result_pass("File-backed THP split to order %d test done\n", order); 400 return; 401 402 close_file: 403 close(fd); 404 cleanup: 405 umount(tmpfs_loc); 406 out: 407 free(file_buf1); 408 free(file_buf2); 409 rmdir(tmpfs_loc); 410 ksft_exit_fail_msg("Error occurred\n"); 411 } 412 413 static bool prepare_thp_fs(const char *xfs_path, char *thp_fs_template, 414 const char **thp_fs_loc) 415 { 416 if (xfs_path) { 417 *thp_fs_loc = xfs_path; 418 return false; 419 } 420 421 *thp_fs_loc = mkdtemp(thp_fs_template); 422 423 if (!*thp_fs_loc) 424 ksft_exit_fail_msg("cannot create temp folder\n"); 425 426 return true; 427 } 428 429 static void cleanup_thp_fs(const char *thp_fs_loc, bool created_tmp) 430 { 431 int status; 432 433 if (!created_tmp) 434 return; 435 436 status = rmdir(thp_fs_loc); 437 if (status) 438 ksft_exit_fail_msg("cannot remove tmp dir: %s\n", 439 strerror(errno)); 440 } 441 442 static int create_pagecache_thp_and_fd(const char *testfile, size_t fd_size, 443 int *fd, char **addr) 444 { 445 size_t i; 446 unsigned char buf[1024]; 447 448 srand(time(NULL)); 449 450 *fd = open(testfile, O_CREAT | O_RDWR, 0664); 451 if (*fd == -1) 452 ksft_exit_fail_msg("Failed to create a file at %s\n", testfile); 453 454 assert(fd_size % sizeof(buf) == 0); 455 for (i = 0; i < sizeof(buf); i++) 456 buf[i] = (unsigned char)i; 457 for (i = 0; i < fd_size; i += sizeof(buf)) { 458 if (write(*fd, buf, sizeof(buf)) != sizeof(buf)) { 459 ksft_perror("write testfile"); 460 close(*fd); 461 goto err_out_unlink; 462 } 463 } 464 close(*fd); 465 sync(); 466 *fd = open("/proc/sys/vm/drop_caches", O_WRONLY); 467 if (*fd == -1) { 468 ksft_perror("open drop_caches"); 469 goto err_out_unlink; 470 } 471 if (write(*fd, "3", 1) != 1) { 472 ksft_perror("write to drop_caches"); 473 goto err_out_close; 474 } 475 close(*fd); 476 477 *fd = open(testfile, O_RDWR); 478 if (*fd == -1) { 479 ksft_perror("Failed to open testfile\n"); 480 goto err_out_unlink; 481 } 482 483 *addr = mmap(NULL, fd_size, PROT_READ|PROT_WRITE, MAP_SHARED, *fd, 0); 484 if (*addr == (char *)-1) { 485 ksft_perror("cannot mmap"); 486 goto err_out_close; 487 } 488 madvise(*addr, fd_size, MADV_HUGEPAGE); 489 490 force_read_pages(*addr, fd_size / pmd_pagesize, pmd_pagesize); 491 492 if (!check_huge_file(*addr, fd_size, fd_size / pmd_pagesize, pmd_pagesize)) { 493 ksft_print_msg("No large pagecache folio generated, please provide a filesystem supporting large folio\n"); 494 munmap(*addr, fd_size); 495 close(*fd); 496 unlink(testfile); 497 ksft_test_result_skip("Pagecache folio split skipped\n"); 498 return -2; 499 } 500 disable_khugepaged(*addr, fd_size); 501 return 0; 502 err_out_close: 503 close(*fd); 504 err_out_unlink: 505 unlink(testfile); 506 ksft_exit_fail_msg("Failed to create large pagecache folios\n"); 507 } 508 509 static void split_thp_in_pagecache_to_order_at(size_t fd_size, 510 const char *fs_loc, int order, int offset) 511 { 512 int fd; 513 char *split_addr; 514 char *addr; 515 size_t i; 516 char testfile[INPUT_MAX]; 517 int err = 0; 518 519 err = snprintf(testfile, INPUT_MAX, "%s/test", fs_loc); 520 521 if (err < 0) 522 ksft_exit_fail_msg("cannot generate right test file name\n"); 523 524 err = create_pagecache_thp_and_fd(testfile, fd_size, &fd, &addr); 525 if (err) 526 return; 527 528 err = 0; 529 530 memset(expected_orders, 0, sizeof(int) * (pmd_order + 1)); 531 /* 532 * use [split_addr, split_addr + pagesize) range to split THPs, since 533 * the debugfs function always split a range with pagesize step and 534 * providing a full [addr, addr + fd_size) range can trigger multiple 535 * splits, complicating after-split result checking. 536 */ 537 if (offset == -1) { 538 for (split_addr = addr; split_addr < addr + fd_size; split_addr += pmd_pagesize) 539 write_debugfs(PID_FMT, getpid(), (uint64_t)split_addr, 540 (uint64_t)split_addr + pagesize, order); 541 542 expected_orders[order] = fd_size / (pagesize << order); 543 } else { 544 int times = fd_size / pmd_pagesize; 545 546 for (split_addr = addr; split_addr < addr + fd_size; split_addr += pmd_pagesize) 547 write_debugfs(PID_FMT_OFFSET, getpid(), (uint64_t)split_addr, 548 (uint64_t)split_addr + pagesize, order, offset); 549 550 for (i = order + 1; i < pmd_order; i++) 551 expected_orders[i] = times; 552 expected_orders[order] = 2 * times; 553 } 554 555 for (i = 0; i < fd_size; i++) 556 if (*(addr + i) != (char)i) { 557 ksft_print_msg("%lu byte corrupted in the file\n", i); 558 err = EXIT_FAILURE; 559 goto out; 560 } 561 562 if (check_after_split_folio_orders(addr, fd_size, pagemap_fd, 563 kpageflags_fd, expected_orders, 564 (pmd_order + 1))) { 565 ksft_print_msg("Unexpected THP split\n"); 566 err = 1; 567 goto out; 568 } 569 570 if (!check_huge_file(addr, fd_size, 0, pmd_pagesize)) { 571 ksft_print_msg("Still FilePmdMapped not split\n"); 572 err = EXIT_FAILURE; 573 goto out; 574 } 575 576 out: 577 munmap(addr, fd_size); 578 close(fd); 579 unlink(testfile); 580 if (offset == -1) { 581 if (err) 582 ksft_exit_fail_msg("Split PMD-mapped pagecache folio to order %d failed\n", order); 583 ksft_test_result_pass("Split PMD-mapped pagecache folio to order %d passed\n", order); 584 } else { 585 if (err) 586 ksft_exit_fail_msg("Split PMD-mapped pagecache folio to order %d at in-folio offset %d failed\n", order, offset); 587 ksft_test_result_pass("Split PMD-mapped pagecache folio to order %d at in-folio offset %d passed\n", order, offset); 588 } 589 } 590 591 int main(int argc, char **argv) 592 { 593 int i; 594 size_t fd_size; 595 char *optional_xfs_path = NULL; 596 char fs_loc_template[] = "/tmp/thp_fs_XXXXXX"; 597 const char *fs_loc; 598 bool created_tmp; 599 int offset; 600 unsigned int nr_pages; 601 unsigned int tests; 602 603 ksft_print_header(); 604 605 if (geteuid() != 0) { 606 ksft_print_msg("Please run the benchmark as root\n"); 607 ksft_finished(); 608 } 609 610 if (!thp_is_enabled()) 611 ksft_exit_skip("Transparent Hugepages not available\n"); 612 613 if (argc > 1) 614 optional_xfs_path = argv[1]; 615 616 pagesize = getpagesize(); 617 pageshift = ffs(pagesize) - 1; 618 pmd_pagesize = read_pmd_pagesize(); 619 if (!pmd_pagesize) 620 ksft_exit_fail_msg("Reading PMD pagesize failed\n"); 621 622 nr_pages = pmd_pagesize / pagesize; 623 pmd_order = sz2ord(pmd_pagesize, pagesize); 624 625 expected_orders = (int *)malloc(sizeof(int) * (pmd_order + 1)); 626 if (!expected_orders) 627 ksft_exit_fail_msg("Fail to allocate memory: %s\n", strerror(errno)); 628 629 tests = 2 + (pmd_order - 1) + (2 * pmd_order) + (pmd_order - 1) * 4 + 2; 630 ksft_set_plan(tests); 631 632 pagemap_fd = open(pagemap_proc, O_RDONLY); 633 if (pagemap_fd == -1) 634 ksft_exit_fail_msg("read pagemap: %s\n", strerror(errno)); 635 636 kpageflags_fd = open(kpageflags_proc, O_RDONLY); 637 if (kpageflags_fd == -1) 638 ksft_exit_fail_msg("read kpageflags: %s\n", strerror(errno)); 639 640 fd_size = 2 * pmd_pagesize; 641 642 split_pmd_zero_pages(); 643 644 for (i = 0; i < pmd_order; i++) 645 if (i != 1) 646 split_pmd_thp_to_order(i); 647 648 split_pte_mapped_thp(); 649 for (i = 0; i < pmd_order; i++) 650 split_file_backed_thp(i); 651 652 created_tmp = prepare_thp_fs(optional_xfs_path, fs_loc_template, 653 &fs_loc); 654 for (i = pmd_order - 1; i >= 0; i--) 655 split_thp_in_pagecache_to_order_at(fd_size, fs_loc, i, -1); 656 657 for (i = 0; i < pmd_order; i++) 658 for (offset = 0; 659 offset < nr_pages; 660 offset += MAX(nr_pages / 4, 1 << i)) 661 split_thp_in_pagecache_to_order_at(fd_size, fs_loc, i, offset); 662 cleanup_thp_fs(fs_loc, created_tmp); 663 664 close(pagemap_fd); 665 close(kpageflags_fd); 666 free(expected_orders); 667 668 ksft_finished(); 669 } 670