1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * A test of splitting PMD THPs and PTE-mapped THPs from a specified virtual 4 * address range in a process via <debugfs>/split_huge_pages interface. 5 */ 6 7 #define _GNU_SOURCE 8 #include <assert.h> 9 #include <stdio.h> 10 #include <stdlib.h> 11 #include <stdarg.h> 12 #include <unistd.h> 13 #include <inttypes.h> 14 #include <string.h> 15 #include <fcntl.h> 16 #include <sys/mman.h> 17 #include <sys/mount.h> 18 #include <sys/param.h> 19 #include <malloc.h> 20 #include <stdbool.h> 21 #include <time.h> 22 #include "vm_util.h" 23 #include "kselftest.h" 24 25 uint64_t pagesize; 26 unsigned int pageshift; 27 uint64_t pmd_pagesize; 28 unsigned int pmd_order; 29 int *expected_orders; 30 31 #define SPLIT_DEBUGFS "/sys/kernel/debug/split_huge_pages" 32 #define SMAP_PATH "/proc/self/smaps" 33 #define INPUT_MAX 80 34 35 #define PID_FMT "%d,0x%lx,0x%lx,%d" 36 #define PID_FMT_OFFSET "%d,0x%lx,0x%lx,%d,%d" 37 #define PATH_FMT "%s,0x%lx,0x%lx,%d" 38 39 const char *pagemap_proc = "/proc/self/pagemap"; 40 const char *kpageflags_proc = "/proc/kpageflags"; 41 int pagemap_fd; 42 int kpageflags_fd; 43 44 static int check_after_split_folio_orders(char *vaddr_start, size_t len, 45 int pagemap_fd, int kpageflags_fd, int orders[], int nr_orders) 46 { 47 int *vaddr_orders; 48 int status; 49 int i; 50 51 vaddr_orders = (int *)malloc(sizeof(int) * nr_orders); 52 53 if (!vaddr_orders) 54 ksft_exit_fail_msg("Cannot allocate memory for vaddr_orders"); 55 56 memset(vaddr_orders, 0, sizeof(int) * nr_orders); 57 status = gather_folio_orders(vaddr_start, len, pagemap_fd, 58 kpageflags_fd, vaddr_orders, nr_orders); 59 if (status) 60 ksft_exit_fail_msg("gather folio info failed\n"); 61 62 for (i = 0; i < nr_orders; i++) 63 if (vaddr_orders[i] != orders[i]) { 64 ksft_print_msg("order %d: expected: %d got %d\n", i, 65 orders[i], vaddr_orders[i]); 66 status = -1; 67 } 68 69 free(vaddr_orders); 70 return status; 71 } 72 73 static void write_debugfs(const char *fmt, ...) 74 { 75 char input[INPUT_MAX]; 76 int ret; 77 va_list argp; 78 79 va_start(argp, fmt); 80 ret = vsnprintf(input, INPUT_MAX, fmt, argp); 81 va_end(argp); 82 83 if (ret >= INPUT_MAX) 84 ksft_exit_fail_msg("%s: Debugfs input is too long\n", __func__); 85 86 ret = write_file(SPLIT_DEBUGFS, input, ret + 1); 87 if (ret) 88 ksft_exit_fail_msg("write_file(%s): %s\n", SPLIT_DEBUGFS, 89 strerror(-ret)); 90 } 91 92 static char *allocate_zero_filled_hugepage(size_t len) 93 { 94 char *result; 95 size_t i; 96 97 result = memalign(pmd_pagesize, len); 98 if (!result) { 99 printf("Fail to allocate memory\n"); 100 exit(EXIT_FAILURE); 101 } 102 103 madvise(result, len, MADV_HUGEPAGE); 104 105 for (i = 0; i < len; i++) 106 result[i] = (char)0; 107 108 return result; 109 } 110 111 static void verify_rss_anon_split_huge_page_all_zeroes(char *one_page, int nr_hpages, size_t len) 112 { 113 unsigned long rss_anon_before, rss_anon_after; 114 size_t i; 115 116 if (!check_huge_anon(one_page, nr_hpages * pmd_pagesize, nr_hpages, pmd_pagesize)) 117 ksft_exit_fail_msg("No THP is allocated\n"); 118 119 rss_anon_before = rss_anon(); 120 if (!rss_anon_before) 121 ksft_exit_fail_msg("No RssAnon is allocated before split\n"); 122 123 /* split all THPs */ 124 write_debugfs(PID_FMT, getpid(), (uint64_t)one_page, 125 (uint64_t)one_page + len, 0); 126 127 for (i = 0; i < len; i++) 128 if (one_page[i] != (char)0) 129 ksft_exit_fail_msg("%ld byte corrupted\n", i); 130 131 if (!check_huge_anon(one_page, nr_hpages * pmd_pagesize, 0, pmd_pagesize)) 132 ksft_exit_fail_msg("Still AnonHugePages not split\n"); 133 134 rss_anon_after = rss_anon(); 135 if (rss_anon_after >= rss_anon_before) 136 ksft_exit_fail_msg("Incorrect RssAnon value. Before: %ld After: %ld\n", 137 rss_anon_before, rss_anon_after); 138 } 139 140 static void split_pmd_zero_pages(void) 141 { 142 char *one_page; 143 int nr_hpages = 4; 144 size_t len = nr_hpages * pmd_pagesize; 145 146 one_page = allocate_zero_filled_hugepage(len); 147 verify_rss_anon_split_huge_page_all_zeroes(one_page, nr_hpages, len); 148 ksft_test_result_pass("Split zero filled huge pages successful\n"); 149 free(one_page); 150 } 151 152 static void split_pmd_thp_to_order(int order) 153 { 154 char *one_page; 155 size_t len = 4 * pmd_pagesize; 156 size_t i; 157 158 one_page = memalign(pmd_pagesize, len); 159 if (!one_page) 160 ksft_exit_fail_msg("Fail to allocate memory: %s\n", strerror(errno)); 161 162 madvise(one_page, len, MADV_HUGEPAGE); 163 164 for (i = 0; i < len; i++) 165 one_page[i] = (char)i; 166 167 if (!check_huge_anon(one_page, 4 * pmd_pagesize, 4, pmd_pagesize)) 168 ksft_exit_fail_msg("No THP is allocated\n"); 169 170 /* split all THPs */ 171 write_debugfs(PID_FMT, getpid(), (uint64_t)one_page, 172 (uint64_t)one_page + len, order); 173 174 for (i = 0; i < len; i++) 175 if (one_page[i] != (char)i) 176 ksft_exit_fail_msg("%ld byte corrupted\n", i); 177 178 memset(expected_orders, 0, sizeof(int) * (pmd_order + 1)); 179 expected_orders[order] = 4 << (pmd_order - order); 180 181 if (check_after_split_folio_orders(one_page, len, pagemap_fd, 182 kpageflags_fd, expected_orders, 183 (pmd_order + 1))) 184 ksft_exit_fail_msg("Unexpected THP split\n"); 185 186 if (!check_huge_anon(one_page, 4 * pmd_pagesize, 0, pmd_pagesize)) 187 ksft_exit_fail_msg("Still AnonHugePages not split\n"); 188 189 ksft_test_result_pass("Split huge pages to order %d successful\n", order); 190 free(one_page); 191 } 192 193 static void split_pte_mapped_thp(void) 194 { 195 const size_t nr_thps = 4; 196 const size_t thp_area_size = nr_thps * pmd_pagesize; 197 const size_t page_area_size = nr_thps * pagesize; 198 char *thp_area, *tmp, *page_area = MAP_FAILED; 199 size_t i; 200 201 thp_area = mmap((void *)(1UL << 30), thp_area_size, PROT_READ | PROT_WRITE, 202 MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); 203 if (thp_area == MAP_FAILED) { 204 ksft_test_result_fail("Fail to allocate memory: %s\n", strerror(errno)); 205 return; 206 } 207 208 madvise(thp_area, thp_area_size, MADV_HUGEPAGE); 209 210 for (i = 0; i < thp_area_size; i++) 211 thp_area[i] = (char)i; 212 213 if (!check_huge_anon(thp_area, nr_thps * pmd_pagesize, nr_thps, pmd_pagesize)) { 214 ksft_test_result_skip("Not all THPs allocated\n"); 215 goto out; 216 } 217 218 /* 219 * To challenge spitting code, we will mremap a single page of each 220 * THP (page[i] of thp[i]) in the thp_area into page_area. This will 221 * replace the PMD mappings in the thp_area by PTE mappings first, 222 * but leaving the THP unsplit, to then create a page-sized hole in 223 * the thp_area. 224 * We will then manually trigger splitting of all THPs through the 225 * single mremap'ed pages of each THP in the page_area. 226 */ 227 page_area = mmap(NULL, page_area_size, PROT_READ | PROT_WRITE, 228 MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); 229 if (page_area == MAP_FAILED) { 230 ksft_test_result_fail("Fail to allocate memory: %s\n", strerror(errno)); 231 goto out; 232 } 233 234 for (i = 0; i < nr_thps; i++) { 235 tmp = mremap(thp_area + pmd_pagesize * i + pagesize * i, 236 pagesize, pagesize, MREMAP_MAYMOVE|MREMAP_FIXED, 237 page_area + pagesize * i); 238 if (tmp != MAP_FAILED) 239 continue; 240 ksft_test_result_fail("mremap failed: %s\n", strerror(errno)); 241 goto out; 242 } 243 244 /* 245 * Verify that our THPs were not split yet. Note that 246 * check_huge_anon() cannot be used as it checks for PMD mappings. 247 */ 248 for (i = 0; i < nr_thps; i++) { 249 if (is_backed_by_folio(page_area + i * pagesize, pmd_order, 250 pagemap_fd, kpageflags_fd)) 251 continue; 252 ksft_test_result_fail("THP %zu missing after mremap\n", i); 253 goto out; 254 } 255 256 /* Split all THPs through the remapped pages. */ 257 write_debugfs(PID_FMT, getpid(), (uint64_t)page_area, 258 (uint64_t)page_area + page_area_size, 0); 259 260 /* Corruption during mremap or split? */ 261 for (i = 0; i < page_area_size; i++) { 262 if (page_area[i] == (char)i) 263 continue; 264 ksft_test_result_fail("%zu byte corrupted\n", i); 265 goto out; 266 } 267 268 /* Split failed? */ 269 for (i = 0; i < nr_thps; i++) { 270 if (is_backed_by_folio(page_area + i * pagesize, 0, 271 pagemap_fd, kpageflags_fd)) 272 continue; 273 ksft_test_result_fail("THP %zu not split\n", i); 274 } 275 276 ksft_test_result_pass("Split PTE-mapped huge pages successful\n"); 277 out: 278 munmap(thp_area, thp_area_size); 279 if (page_area != MAP_FAILED) 280 munmap(page_area, page_area_size); 281 } 282 283 static void split_file_backed_thp(int order) 284 { 285 int status; 286 int fd; 287 char tmpfs_template[] = "/tmp/thp_split_XXXXXX"; 288 const char *tmpfs_loc = mkdtemp(tmpfs_template); 289 char testfile[INPUT_MAX]; 290 unsigned long size = 2 * pmd_pagesize; 291 char opts[64]; 292 ssize_t num_written, num_read; 293 char *file_buf1 = NULL, *file_buf2 = NULL; 294 uint64_t pgoff_start = 0, pgoff_end = 1024; 295 int i; 296 297 ksft_print_msg("Please enable pr_debug in split_huge_pages_in_file() for more info.\n"); 298 299 if (!tmpfs_loc) 300 ksft_exit_fail_msg("mkdtemp failed\n"); 301 302 file_buf1 = (char *)malloc(pmd_pagesize); 303 file_buf2 = (char *)malloc(pmd_pagesize); 304 305 if (!file_buf1 || !file_buf2) { 306 ksft_print_msg("cannot allocate file buffers\n"); 307 goto out; 308 } 309 310 for (i = 0; i < pmd_pagesize; i++) 311 file_buf1[i] = (char)i; 312 memset(file_buf2, 0, pmd_pagesize); 313 314 snprintf(opts, sizeof(opts), "huge=always,size=%lu", size); 315 status = mount("tmpfs", tmpfs_loc, "tmpfs", 0, opts); 316 317 if (status) { 318 ksft_print_msg("Unable to create a tmpfs for testing\n"); 319 goto out; 320 } 321 322 status = snprintf(testfile, INPUT_MAX, "%s/thp_file", tmpfs_loc); 323 if (status >= INPUT_MAX) { 324 ksft_print_msg("Fail to create file-backed THP split testing file\n"); 325 goto cleanup; 326 } 327 328 fd = open(testfile, O_CREAT|O_RDWR, 0664); 329 if (fd == -1) { 330 ksft_perror("Cannot open testing file"); 331 goto cleanup; 332 } 333 334 /* write pmd size data to the file, so a file-backed THP can be allocated */ 335 num_written = write(fd, file_buf1, pmd_pagesize); 336 337 if (num_written == -1 || num_written != pmd_pagesize) { 338 ksft_perror("Failed to write data to testing file"); 339 goto close_file; 340 } 341 342 /* split the file-backed THP */ 343 write_debugfs(PATH_FMT, testfile, pgoff_start, pgoff_end, order); 344 345 /* check file content after split */ 346 status = lseek(fd, 0, SEEK_SET); 347 if (status == -1) { 348 ksft_perror("Cannot lseek file"); 349 goto close_file; 350 } 351 352 num_read = read(fd, file_buf2, num_written); 353 if (num_read == -1 || num_read != num_written) { 354 ksft_perror("Cannot read file content back"); 355 goto close_file; 356 } 357 358 if (strncmp(file_buf1, file_buf2, pmd_pagesize) != 0) { 359 ksft_print_msg("File content changed\n"); 360 goto close_file; 361 } 362 363 close(fd); 364 status = unlink(testfile); 365 if (status) { 366 ksft_perror("Cannot remove testing file"); 367 goto cleanup; 368 } 369 370 status = umount(tmpfs_loc); 371 if (status) { 372 ksft_print_msg("Unable to umount %s\n", tmpfs_loc); 373 goto out; 374 } 375 376 free(file_buf1); 377 free(file_buf2); 378 379 status = rmdir(tmpfs_loc); 380 if (status) 381 ksft_exit_fail_msg("cannot remove tmp dir: %s\n", strerror(errno)); 382 383 ksft_print_msg("Please check dmesg for more information\n"); 384 ksft_test_result_pass("File-backed THP split to order %d test done\n", order); 385 return; 386 387 close_file: 388 close(fd); 389 cleanup: 390 umount(tmpfs_loc); 391 out: 392 free(file_buf1); 393 free(file_buf2); 394 rmdir(tmpfs_loc); 395 ksft_exit_fail_msg("Error occurred\n"); 396 } 397 398 static bool prepare_thp_fs(const char *xfs_path, char *thp_fs_template, 399 const char **thp_fs_loc) 400 { 401 if (xfs_path) { 402 *thp_fs_loc = xfs_path; 403 return false; 404 } 405 406 *thp_fs_loc = mkdtemp(thp_fs_template); 407 408 if (!*thp_fs_loc) 409 ksft_exit_fail_msg("cannot create temp folder\n"); 410 411 return true; 412 } 413 414 static void cleanup_thp_fs(const char *thp_fs_loc, bool created_tmp) 415 { 416 int status; 417 418 if (!created_tmp) 419 return; 420 421 status = rmdir(thp_fs_loc); 422 if (status) 423 ksft_exit_fail_msg("cannot remove tmp dir: %s\n", 424 strerror(errno)); 425 } 426 427 static int create_pagecache_thp_and_fd(const char *testfile, size_t fd_size, 428 int *fd, char **addr) 429 { 430 size_t i; 431 unsigned char buf[1024]; 432 433 srand(time(NULL)); 434 435 *fd = open(testfile, O_CREAT | O_RDWR, 0664); 436 if (*fd == -1) 437 ksft_exit_fail_msg("Failed to create a file at %s\n", testfile); 438 439 assert(fd_size % sizeof(buf) == 0); 440 for (i = 0; i < sizeof(buf); i++) 441 buf[i] = (unsigned char)i; 442 for (i = 0; i < fd_size; i += sizeof(buf)) { 443 if (write(*fd, buf, sizeof(buf)) != sizeof(buf)) { 444 ksft_perror("write testfile"); 445 close(*fd); 446 goto err_out_unlink; 447 } 448 } 449 close(*fd); 450 sync(); 451 *fd = open("/proc/sys/vm/drop_caches", O_WRONLY); 452 if (*fd == -1) { 453 ksft_perror("open drop_caches"); 454 goto err_out_unlink; 455 } 456 if (write(*fd, "3", 1) != 1) { 457 ksft_perror("write to drop_caches"); 458 goto err_out_close; 459 } 460 close(*fd); 461 462 *fd = open(testfile, O_RDWR); 463 if (*fd == -1) { 464 ksft_perror("Failed to open testfile\n"); 465 goto err_out_unlink; 466 } 467 468 *addr = mmap(NULL, fd_size, PROT_READ|PROT_WRITE, MAP_SHARED, *fd, 0); 469 if (*addr == (char *)-1) { 470 ksft_perror("cannot mmap"); 471 goto err_out_close; 472 } 473 madvise(*addr, fd_size, MADV_HUGEPAGE); 474 475 force_read_pages(*addr, fd_size / pmd_pagesize, pmd_pagesize); 476 477 if (!check_huge_file(*addr, fd_size, fd_size / pmd_pagesize, pmd_pagesize)) { 478 ksft_print_msg("No large pagecache folio generated, please provide a filesystem supporting large folio\n"); 479 munmap(*addr, fd_size); 480 close(*fd); 481 unlink(testfile); 482 ksft_test_result_skip("Pagecache folio split skipped\n"); 483 return -2; 484 } 485 return 0; 486 err_out_close: 487 close(*fd); 488 err_out_unlink: 489 unlink(testfile); 490 ksft_exit_fail_msg("Failed to create large pagecache folios\n"); 491 } 492 493 static void split_thp_in_pagecache_to_order_at(size_t fd_size, 494 const char *fs_loc, int order, int offset) 495 { 496 int fd; 497 char *split_addr; 498 char *addr; 499 size_t i; 500 char testfile[INPUT_MAX]; 501 int err = 0; 502 503 err = snprintf(testfile, INPUT_MAX, "%s/test", fs_loc); 504 505 if (err < 0) 506 ksft_exit_fail_msg("cannot generate right test file name\n"); 507 508 err = create_pagecache_thp_and_fd(testfile, fd_size, &fd, &addr); 509 if (err) 510 return; 511 512 err = 0; 513 514 memset(expected_orders, 0, sizeof(int) * (pmd_order + 1)); 515 /* 516 * use [split_addr, split_addr + pagesize) range to split THPs, since 517 * the debugfs function always split a range with pagesize step and 518 * providing a full [addr, addr + fd_size) range can trigger multiple 519 * splits, complicating after-split result checking. 520 */ 521 if (offset == -1) { 522 for (split_addr = addr; split_addr < addr + fd_size; split_addr += pmd_pagesize) 523 write_debugfs(PID_FMT, getpid(), (uint64_t)split_addr, 524 (uint64_t)split_addr + pagesize, order); 525 526 expected_orders[order] = fd_size / (pagesize << order); 527 } else { 528 int times = fd_size / pmd_pagesize; 529 530 for (split_addr = addr; split_addr < addr + fd_size; split_addr += pmd_pagesize) 531 write_debugfs(PID_FMT_OFFSET, getpid(), (uint64_t)split_addr, 532 (uint64_t)split_addr + pagesize, order, offset); 533 534 for (i = order + 1; i < pmd_order; i++) 535 expected_orders[i] = times; 536 expected_orders[order] = 2 * times; 537 } 538 539 for (i = 0; i < fd_size; i++) 540 if (*(addr + i) != (char)i) { 541 ksft_print_msg("%lu byte corrupted in the file\n", i); 542 err = EXIT_FAILURE; 543 goto out; 544 } 545 546 if (check_after_split_folio_orders(addr, fd_size, pagemap_fd, 547 kpageflags_fd, expected_orders, 548 (pmd_order + 1))) { 549 ksft_print_msg("Unexpected THP split\n"); 550 err = 1; 551 goto out; 552 } 553 554 if (!check_huge_file(addr, fd_size, 0, pmd_pagesize)) { 555 ksft_print_msg("Still FilePmdMapped not split\n"); 556 err = EXIT_FAILURE; 557 goto out; 558 } 559 560 out: 561 munmap(addr, fd_size); 562 close(fd); 563 unlink(testfile); 564 if (offset == -1) { 565 if (err) 566 ksft_exit_fail_msg("Split PMD-mapped pagecache folio to order %d failed\n", order); 567 ksft_test_result_pass("Split PMD-mapped pagecache folio to order %d passed\n", order); 568 } else { 569 if (err) 570 ksft_exit_fail_msg("Split PMD-mapped pagecache folio to order %d at in-folio offset %d failed\n", order, offset); 571 ksft_test_result_pass("Split PMD-mapped pagecache folio to order %d at in-folio offset %d passed\n", order, offset); 572 } 573 } 574 575 int main(int argc, char **argv) 576 { 577 int i; 578 size_t fd_size; 579 char *optional_xfs_path = NULL; 580 char fs_loc_template[] = "/tmp/thp_fs_XXXXXX"; 581 const char *fs_loc; 582 bool created_tmp; 583 int offset; 584 unsigned int nr_pages; 585 unsigned int tests; 586 587 ksft_print_header(); 588 589 if (geteuid() != 0) { 590 ksft_print_msg("Please run the benchmark as root\n"); 591 ksft_finished(); 592 } 593 594 if (!thp_is_enabled()) 595 ksft_exit_skip("Transparent Hugepages not available\n"); 596 597 if (argc > 1) 598 optional_xfs_path = argv[1]; 599 600 pagesize = getpagesize(); 601 pageshift = ffs(pagesize) - 1; 602 pmd_pagesize = read_pmd_pagesize(); 603 if (!pmd_pagesize) 604 ksft_exit_fail_msg("Reading PMD pagesize failed\n"); 605 606 nr_pages = pmd_pagesize / pagesize; 607 pmd_order = sz2ord(pmd_pagesize, pagesize); 608 609 expected_orders = (int *)malloc(sizeof(int) * (pmd_order + 1)); 610 if (!expected_orders) 611 ksft_exit_fail_msg("Fail to allocate memory: %s\n", strerror(errno)); 612 613 tests = 2 + (pmd_order - 1) + (2 * pmd_order) + (pmd_order - 1) * 4 + 2; 614 ksft_set_plan(tests); 615 616 pagemap_fd = open(pagemap_proc, O_RDONLY); 617 if (pagemap_fd == -1) 618 ksft_exit_fail_msg("read pagemap: %s\n", strerror(errno)); 619 620 kpageflags_fd = open(kpageflags_proc, O_RDONLY); 621 if (kpageflags_fd == -1) 622 ksft_exit_fail_msg("read kpageflags: %s\n", strerror(errno)); 623 624 fd_size = 2 * pmd_pagesize; 625 626 split_pmd_zero_pages(); 627 628 for (i = 0; i < pmd_order; i++) 629 if (i != 1) 630 split_pmd_thp_to_order(i); 631 632 split_pte_mapped_thp(); 633 for (i = 0; i < pmd_order; i++) 634 split_file_backed_thp(i); 635 636 created_tmp = prepare_thp_fs(optional_xfs_path, fs_loc_template, 637 &fs_loc); 638 for (i = pmd_order - 1; i >= 0; i--) 639 split_thp_in_pagecache_to_order_at(fd_size, fs_loc, i, -1); 640 641 for (i = 0; i < pmd_order; i++) 642 for (offset = 0; 643 offset < nr_pages; 644 offset += MAX(nr_pages / 4, 1 << i)) 645 split_thp_in_pagecache_to_order_at(fd_size, fs_loc, i, offset); 646 cleanup_thp_fs(fs_loc, created_tmp); 647 648 close(pagemap_fd); 649 close(kpageflags_fd); 650 free(expected_orders); 651 652 ksft_finished(); 653 } 654