1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * KSM functional tests 4 * 5 * Copyright 2022, Red Hat, Inc. 6 * 7 * Author(s): David Hildenbrand <david@redhat.com> 8 */ 9 #define _GNU_SOURCE 10 #include <stdlib.h> 11 #include <string.h> 12 #include <stdbool.h> 13 #include <stdint.h> 14 #include <unistd.h> 15 #include <errno.h> 16 #include <fcntl.h> 17 #include <sys/mman.h> 18 #include <sys/prctl.h> 19 #include <sys/syscall.h> 20 #include <sys/ioctl.h> 21 #include <sys/wait.h> 22 #include <linux/userfaultfd.h> 23 24 #include "../kselftest.h" 25 #include "vm_util.h" 26 27 #define KiB 1024u 28 #define MiB (1024 * KiB) 29 #define FORK_EXEC_CHILD_PRG_NAME "ksm_fork_exec_child" 30 31 #define MAP_MERGE_FAIL ((void *)-1) 32 #define MAP_MERGE_SKIP ((void *)-2) 33 34 enum ksm_merge_mode { 35 KSM_MERGE_PRCTL, 36 KSM_MERGE_MADVISE, 37 KSM_MERGE_NONE, /* PRCTL already set */ 38 }; 39 40 static int mem_fd; 41 static int pagemap_fd; 42 static size_t pagesize; 43 44 static void init_global_file_handles(void); 45 46 static bool range_maps_duplicates(char *addr, unsigned long size) 47 { 48 unsigned long offs_a, offs_b, pfn_a, pfn_b; 49 50 /* 51 * There is no easy way to check if there are KSM pages mapped into 52 * this range. We only check that the range does not map the same PFN 53 * twice by comparing each pair of mapped pages. 54 */ 55 for (offs_a = 0; offs_a < size; offs_a += pagesize) { 56 pfn_a = pagemap_get_pfn(pagemap_fd, addr + offs_a); 57 /* Page not present or PFN not exposed by the kernel. */ 58 if (pfn_a == -1ul || !pfn_a) 59 continue; 60 61 for (offs_b = offs_a + pagesize; offs_b < size; 62 offs_b += pagesize) { 63 pfn_b = pagemap_get_pfn(pagemap_fd, addr + offs_b); 64 if (pfn_b == -1ul || !pfn_b) 65 continue; 66 if (pfn_a == pfn_b) 67 return true; 68 } 69 } 70 return false; 71 } 72 73 static char *__mmap_and_merge_range(char val, unsigned long size, int prot, 74 enum ksm_merge_mode mode) 75 { 76 char *map; 77 char *err_map = MAP_MERGE_FAIL; 78 int ret; 79 80 /* Stabilize accounting by disabling KSM completely. */ 81 if (ksm_stop() < 0) { 82 ksft_print_msg("Disabling (unmerging) KSM failed\n"); 83 return err_map; 84 } 85 86 if (ksm_get_self_merging_pages() > 0) { 87 ksft_print_msg("Still pages merged\n"); 88 return err_map; 89 } 90 91 map = mmap(NULL, size, PROT_READ|PROT_WRITE, 92 MAP_PRIVATE|MAP_ANON, -1, 0); 93 if (map == MAP_FAILED) { 94 ksft_print_msg("mmap() failed\n"); 95 return err_map; 96 } 97 98 /* Don't use THP. Ignore if THP are not around on a kernel. */ 99 if (madvise(map, size, MADV_NOHUGEPAGE) && errno != EINVAL) { 100 ksft_print_msg("MADV_NOHUGEPAGE failed\n"); 101 goto unmap; 102 } 103 104 /* Make sure each page contains the same values to merge them. */ 105 memset(map, val, size); 106 107 if (mprotect(map, size, prot)) { 108 ksft_print_msg("mprotect() failed\n"); 109 err_map = MAP_MERGE_SKIP; 110 goto unmap; 111 } 112 113 switch (mode) { 114 case KSM_MERGE_PRCTL: 115 ret = prctl(PR_SET_MEMORY_MERGE, 1, 0, 0, 0); 116 if (ret < 0 && errno == EINVAL) { 117 ksft_print_msg("PR_SET_MEMORY_MERGE not supported\n"); 118 err_map = MAP_MERGE_SKIP; 119 goto unmap; 120 } else if (ret) { 121 ksft_print_msg("PR_SET_MEMORY_MERGE=1 failed\n"); 122 goto unmap; 123 } 124 break; 125 case KSM_MERGE_MADVISE: 126 if (madvise(map, size, MADV_MERGEABLE)) { 127 ksft_print_msg("MADV_MERGEABLE failed\n"); 128 goto unmap; 129 } 130 break; 131 case KSM_MERGE_NONE: 132 break; 133 } 134 135 /* Run KSM to trigger merging and wait. */ 136 if (ksm_start() < 0) { 137 ksft_print_msg("Running KSM failed\n"); 138 goto unmap; 139 } 140 141 /* 142 * Check if anything was merged at all. Ignore the zero page that is 143 * accounted differently (depending on kernel support). 144 */ 145 if (val && !ksm_get_self_merging_pages()) { 146 ksft_print_msg("No pages got merged\n"); 147 goto unmap; 148 } 149 150 return map; 151 unmap: 152 munmap(map, size); 153 return err_map; 154 } 155 156 static char *mmap_and_merge_range(char val, unsigned long size, int prot, 157 enum ksm_merge_mode mode) 158 { 159 char *map; 160 char *ret = MAP_FAILED; 161 162 map = __mmap_and_merge_range(val, size, prot, mode); 163 if (map == MAP_MERGE_FAIL) 164 ksft_test_result_fail("Merging memory failed"); 165 else if (map == MAP_MERGE_SKIP) 166 ksft_test_result_skip("Merging memory skipped"); 167 else 168 ret = map; 169 170 return ret; 171 } 172 173 static void test_unmerge(void) 174 { 175 const unsigned int size = 2 * MiB; 176 char *map; 177 178 ksft_print_msg("[RUN] %s\n", __func__); 179 180 map = mmap_and_merge_range(0xcf, size, PROT_READ | PROT_WRITE, KSM_MERGE_MADVISE); 181 if (map == MAP_FAILED) 182 return; 183 184 if (madvise(map, size, MADV_UNMERGEABLE)) { 185 ksft_test_result_fail("MADV_UNMERGEABLE failed\n"); 186 goto unmap; 187 } 188 189 ksft_test_result(!range_maps_duplicates(map, size), 190 "Pages were unmerged\n"); 191 unmap: 192 ksm_stop(); 193 munmap(map, size); 194 } 195 196 static void test_unmerge_zero_pages(void) 197 { 198 const unsigned int size = 2 * MiB; 199 char *map; 200 unsigned int offs; 201 unsigned long pages_expected; 202 203 ksft_print_msg("[RUN] %s\n", __func__); 204 205 if (ksm_get_self_zero_pages() < 0) { 206 ksft_test_result_skip("accessing \"/proc/self/ksm_stat\" failed\n"); 207 return; 208 } 209 210 if (ksm_use_zero_pages() < 0) { 211 ksft_test_result_skip("write \"/sys/kernel/mm/ksm/use_zero_pages\" failed\n"); 212 return; 213 } 214 215 /* Let KSM deduplicate zero pages. */ 216 map = mmap_and_merge_range(0x00, size, PROT_READ | PROT_WRITE, KSM_MERGE_MADVISE); 217 if (map == MAP_FAILED) 218 return; 219 220 /* Check if ksm_zero_pages is updated correctly after KSM merging */ 221 pages_expected = size / pagesize; 222 if (pages_expected != ksm_get_self_zero_pages()) { 223 ksft_test_result_fail("'ksm_zero_pages' updated after merging\n"); 224 goto unmap; 225 } 226 227 /* Try to unmerge half of the region */ 228 if (madvise(map, size / 2, MADV_UNMERGEABLE)) { 229 ksft_test_result_fail("MADV_UNMERGEABLE failed\n"); 230 goto unmap; 231 } 232 233 /* Check if ksm_zero_pages is updated correctly after unmerging */ 234 pages_expected /= 2; 235 if (pages_expected != ksm_get_self_zero_pages()) { 236 ksft_test_result_fail("'ksm_zero_pages' updated after unmerging\n"); 237 goto unmap; 238 } 239 240 /* Trigger unmerging of the other half by writing to the pages. */ 241 for (offs = size / 2; offs < size; offs += pagesize) 242 *((unsigned int *)&map[offs]) = offs; 243 244 /* Now we should have no zeropages remaining. */ 245 if (ksm_get_self_zero_pages()) { 246 ksft_test_result_fail("'ksm_zero_pages' updated after write fault\n"); 247 goto unmap; 248 } 249 250 /* Check if ksm zero pages are really unmerged */ 251 ksft_test_result(!range_maps_duplicates(map, size), 252 "KSM zero pages were unmerged\n"); 253 unmap: 254 ksm_stop(); 255 munmap(map, size); 256 } 257 258 static void test_unmerge_discarded(void) 259 { 260 const unsigned int size = 2 * MiB; 261 char *map; 262 263 ksft_print_msg("[RUN] %s\n", __func__); 264 265 map = mmap_and_merge_range(0xcf, size, PROT_READ | PROT_WRITE, KSM_MERGE_MADVISE); 266 if (map == MAP_FAILED) 267 return; 268 269 /* Discard half of all mapped pages so we have pte_none() entries. */ 270 if (madvise(map, size / 2, MADV_DONTNEED)) { 271 ksft_test_result_fail("MADV_DONTNEED failed\n"); 272 goto unmap; 273 } 274 275 if (madvise(map, size, MADV_UNMERGEABLE)) { 276 ksft_test_result_fail("MADV_UNMERGEABLE failed\n"); 277 goto unmap; 278 } 279 280 ksft_test_result(!range_maps_duplicates(map, size), 281 "Pages were unmerged\n"); 282 unmap: 283 ksm_stop(); 284 munmap(map, size); 285 } 286 287 #ifdef __NR_userfaultfd 288 static void test_unmerge_uffd_wp(void) 289 { 290 struct uffdio_writeprotect uffd_writeprotect; 291 const unsigned int size = 2 * MiB; 292 struct uffdio_api uffdio_api; 293 char *map; 294 int uffd; 295 296 ksft_print_msg("[RUN] %s\n", __func__); 297 298 map = mmap_and_merge_range(0xcf, size, PROT_READ | PROT_WRITE, KSM_MERGE_MADVISE); 299 if (map == MAP_FAILED) 300 return; 301 302 /* See if UFFD is around. */ 303 uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK); 304 if (uffd < 0) { 305 ksft_test_result_skip("__NR_userfaultfd failed\n"); 306 goto unmap; 307 } 308 309 /* See if UFFD-WP is around. */ 310 uffdio_api.api = UFFD_API; 311 uffdio_api.features = 0; 312 if (ioctl(uffd, UFFDIO_API, &uffdio_api) < 0) { 313 if (errno == EINVAL) 314 ksft_test_result_skip("The API version requested is not supported\n"); 315 else 316 ksft_test_result_fail("UFFDIO_API failed: %s\n", strerror(errno)); 317 318 goto close_uffd; 319 } 320 if (!(uffdio_api.features & UFFD_FEATURE_PAGEFAULT_FLAG_WP)) { 321 ksft_test_result_skip("UFFD_FEATURE_PAGEFAULT_FLAG_WP not available\n"); 322 goto close_uffd; 323 } 324 325 /* 326 * UFFDIO_API must only be called once to enable features. 327 * So we close the old userfaultfd and create a new one to 328 * actually enable UFFD_FEATURE_PAGEFAULT_FLAG_WP. 329 */ 330 close(uffd); 331 uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK); 332 if (uffd < 0) { 333 ksft_test_result_fail("__NR_userfaultfd failed\n"); 334 goto unmap; 335 } 336 337 /* Now, enable it ("two-step handshake") */ 338 uffdio_api.api = UFFD_API; 339 uffdio_api.features = UFFD_FEATURE_PAGEFAULT_FLAG_WP; 340 if (ioctl(uffd, UFFDIO_API, &uffdio_api) < 0) { 341 ksft_test_result_fail("UFFDIO_API failed: %s\n", strerror(errno)); 342 goto close_uffd; 343 } 344 345 /* Register UFFD-WP, no need for an actual handler. */ 346 if (uffd_register(uffd, map, size, false, true, false)) { 347 ksft_test_result_fail("UFFDIO_REGISTER_MODE_WP failed\n"); 348 goto close_uffd; 349 } 350 351 /* Write-protect the range using UFFD-WP. */ 352 uffd_writeprotect.range.start = (unsigned long) map; 353 uffd_writeprotect.range.len = size; 354 uffd_writeprotect.mode = UFFDIO_WRITEPROTECT_MODE_WP; 355 if (ioctl(uffd, UFFDIO_WRITEPROTECT, &uffd_writeprotect)) { 356 ksft_test_result_fail("UFFDIO_WRITEPROTECT failed\n"); 357 goto close_uffd; 358 } 359 360 if (madvise(map, size, MADV_UNMERGEABLE)) { 361 ksft_test_result_fail("MADV_UNMERGEABLE failed\n"); 362 goto close_uffd; 363 } 364 365 ksft_test_result(!range_maps_duplicates(map, size), 366 "Pages were unmerged\n"); 367 close_uffd: 368 close(uffd); 369 unmap: 370 ksm_stop(); 371 munmap(map, size); 372 } 373 #endif 374 375 /* Verify that KSM can be enabled / queried with prctl. */ 376 static void test_prctl(void) 377 { 378 int ret; 379 380 ksft_print_msg("[RUN] %s\n", __func__); 381 382 ret = prctl(PR_SET_MEMORY_MERGE, 1, 0, 0, 0); 383 if (ret < 0 && errno == EINVAL) { 384 ksft_test_result_skip("PR_SET_MEMORY_MERGE not supported\n"); 385 return; 386 } else if (ret) { 387 ksft_test_result_fail("PR_SET_MEMORY_MERGE=1 failed\n"); 388 return; 389 } 390 391 ret = prctl(PR_GET_MEMORY_MERGE, 0, 0, 0, 0); 392 if (ret < 0) { 393 ksft_test_result_fail("PR_GET_MEMORY_MERGE failed\n"); 394 return; 395 } else if (ret != 1) { 396 ksft_test_result_fail("PR_SET_MEMORY_MERGE=1 not effective\n"); 397 return; 398 } 399 400 ret = prctl(PR_SET_MEMORY_MERGE, 0, 0, 0, 0); 401 if (ret) { 402 ksft_test_result_fail("PR_SET_MEMORY_MERGE=0 failed\n"); 403 return; 404 } 405 406 ret = prctl(PR_GET_MEMORY_MERGE, 0, 0, 0, 0); 407 if (ret < 0) { 408 ksft_test_result_fail("PR_GET_MEMORY_MERGE failed\n"); 409 return; 410 } else if (ret != 0) { 411 ksft_test_result_fail("PR_SET_MEMORY_MERGE=0 not effective\n"); 412 return; 413 } 414 415 ksft_test_result_pass("Setting/clearing PR_SET_MEMORY_MERGE works\n"); 416 } 417 418 static int test_child_ksm(void) 419 { 420 const unsigned int size = 2 * MiB; 421 char *map; 422 423 /* Test if KSM is enabled for the process. */ 424 if (prctl(PR_GET_MEMORY_MERGE, 0, 0, 0, 0) != 1) 425 return 1; 426 427 /* Test if merge could really happen. */ 428 map = __mmap_and_merge_range(0xcf, size, PROT_READ | PROT_WRITE, KSM_MERGE_NONE); 429 if (map == MAP_MERGE_FAIL) 430 return 2; 431 else if (map == MAP_MERGE_SKIP) 432 return 3; 433 434 ksm_stop(); 435 munmap(map, size); 436 return 0; 437 } 438 439 static void test_child_ksm_err(int status) 440 { 441 if (status == 1) 442 ksft_test_result_fail("unexpected PR_GET_MEMORY_MERGE result in child\n"); 443 else if (status == 2) 444 ksft_test_result_fail("Merge in child failed\n"); 445 else if (status == 3) 446 ksft_test_result_skip("Merge in child skipped\n"); 447 else if (status == 4) 448 ksft_test_result_fail("Binary not found\n"); 449 } 450 451 /* Verify that prctl ksm flag is inherited. */ 452 static void test_prctl_fork(void) 453 { 454 int ret, status; 455 pid_t child_pid; 456 457 ksft_print_msg("[RUN] %s\n", __func__); 458 459 ret = prctl(PR_SET_MEMORY_MERGE, 1, 0, 0, 0); 460 if (ret < 0 && errno == EINVAL) { 461 ksft_test_result_skip("PR_SET_MEMORY_MERGE not supported\n"); 462 return; 463 } else if (ret) { 464 ksft_test_result_fail("PR_SET_MEMORY_MERGE=1 failed\n"); 465 return; 466 } 467 468 child_pid = fork(); 469 if (!child_pid) { 470 init_global_file_handles(); 471 exit(test_child_ksm()); 472 } else if (child_pid < 0) { 473 ksft_test_result_fail("fork() failed\n"); 474 return; 475 } 476 477 if (waitpid(child_pid, &status, 0) < 0) { 478 ksft_test_result_fail("waitpid() failed\n"); 479 return; 480 } 481 482 status = WEXITSTATUS(status); 483 if (status) { 484 test_child_ksm_err(status); 485 return; 486 } 487 488 if (prctl(PR_SET_MEMORY_MERGE, 0, 0, 0, 0)) { 489 ksft_test_result_fail("PR_SET_MEMORY_MERGE=0 failed\n"); 490 return; 491 } 492 493 ksft_test_result_pass("PR_SET_MEMORY_MERGE value is inherited\n"); 494 } 495 496 static void test_prctl_fork_exec(void) 497 { 498 int ret, status; 499 pid_t child_pid; 500 501 ksft_print_msg("[RUN] %s\n", __func__); 502 503 ret = prctl(PR_SET_MEMORY_MERGE, 1, 0, 0, 0); 504 if (ret < 0 && errno == EINVAL) { 505 ksft_test_result_skip("PR_SET_MEMORY_MERGE not supported\n"); 506 return; 507 } else if (ret) { 508 ksft_test_result_fail("PR_SET_MEMORY_MERGE=1 failed\n"); 509 return; 510 } 511 512 child_pid = fork(); 513 if (child_pid == -1) { 514 ksft_test_result_skip("fork() failed\n"); 515 return; 516 } else if (child_pid == 0) { 517 char *prg_name = "./ksm_functional_tests"; 518 char *argv_for_program[] = { prg_name, FORK_EXEC_CHILD_PRG_NAME, NULL }; 519 520 execv(prg_name, argv_for_program); 521 exit(4); 522 } 523 524 if (waitpid(child_pid, &status, 0) > 0) { 525 if (WIFEXITED(status)) { 526 status = WEXITSTATUS(status); 527 if (status) { 528 test_child_ksm_err(status); 529 return; 530 } 531 } else { 532 ksft_test_result_fail("program didn't terminate normally\n"); 533 return; 534 } 535 } else { 536 ksft_test_result_fail("waitpid() failed\n"); 537 return; 538 } 539 540 if (prctl(PR_SET_MEMORY_MERGE, 0, 0, 0, 0)) { 541 ksft_test_result_fail("PR_SET_MEMORY_MERGE=0 failed\n"); 542 return; 543 } 544 545 ksft_test_result_pass("PR_SET_MEMORY_MERGE value is inherited\n"); 546 } 547 548 static void test_prctl_unmerge(void) 549 { 550 const unsigned int size = 2 * MiB; 551 char *map; 552 553 ksft_print_msg("[RUN] %s\n", __func__); 554 555 map = mmap_and_merge_range(0xcf, size, PROT_READ | PROT_WRITE, KSM_MERGE_PRCTL); 556 if (map == MAP_FAILED) 557 return; 558 559 if (prctl(PR_SET_MEMORY_MERGE, 0, 0, 0, 0)) { 560 ksft_test_result_fail("PR_SET_MEMORY_MERGE=0 failed\n"); 561 goto unmap; 562 } 563 564 ksft_test_result(!range_maps_duplicates(map, size), 565 "Pages were unmerged\n"); 566 unmap: 567 ksm_stop(); 568 munmap(map, size); 569 } 570 571 static void test_prot_none(void) 572 { 573 const unsigned int size = 2 * MiB; 574 char *map; 575 int i; 576 577 ksft_print_msg("[RUN] %s\n", __func__); 578 579 map = mmap_and_merge_range(0x11, size, PROT_NONE, KSM_MERGE_MADVISE); 580 if (map == MAP_FAILED) 581 goto unmap; 582 583 /* Store a unique value in each page on one half using ptrace */ 584 for (i = 0; i < size / 2; i += pagesize) { 585 lseek(mem_fd, (uintptr_t) map + i, SEEK_SET); 586 if (write(mem_fd, &i, sizeof(i)) != sizeof(i)) { 587 ksft_test_result_fail("ptrace write failed\n"); 588 goto unmap; 589 } 590 } 591 592 /* Trigger unsharing on the other half. */ 593 if (madvise(map + size / 2, size / 2, MADV_UNMERGEABLE)) { 594 ksft_test_result_fail("MADV_UNMERGEABLE failed\n"); 595 goto unmap; 596 } 597 598 ksft_test_result(!range_maps_duplicates(map, size), 599 "Pages were unmerged\n"); 600 unmap: 601 ksm_stop(); 602 munmap(map, size); 603 } 604 605 static void init_global_file_handles(void) 606 { 607 mem_fd = open("/proc/self/mem", O_RDWR); 608 if (mem_fd < 0) 609 ksft_exit_fail_msg("opening /proc/self/mem failed\n"); 610 if (ksm_stop() < 0) 611 ksft_exit_skip("accessing \"/sys/kernel/mm/ksm/run\") failed\n"); 612 if (ksm_get_full_scans() < 0) 613 ksft_exit_skip("accessing \"/sys/kernel/mm/ksm/full_scans\") failed\n"); 614 pagemap_fd = open("/proc/self/pagemap", O_RDONLY); 615 if (pagemap_fd < 0) 616 ksft_exit_skip("open(\"/proc/self/pagemap\") failed\n"); 617 if (ksm_get_self_merging_pages() < 0) 618 ksft_exit_skip("accessing \"/proc/self/ksm_merging_pages\") failed\n"); 619 } 620 621 int main(int argc, char **argv) 622 { 623 unsigned int tests = 8; 624 int err; 625 626 if (argc > 1 && !strcmp(argv[1], FORK_EXEC_CHILD_PRG_NAME)) { 627 init_global_file_handles(); 628 exit(test_child_ksm()); 629 } 630 631 #ifdef __NR_userfaultfd 632 tests++; 633 #endif 634 635 ksft_print_header(); 636 ksft_set_plan(tests); 637 638 pagesize = getpagesize(); 639 640 init_global_file_handles(); 641 642 test_unmerge(); 643 test_unmerge_zero_pages(); 644 test_unmerge_discarded(); 645 #ifdef __NR_userfaultfd 646 test_unmerge_uffd_wp(); 647 #endif 648 649 test_prot_none(); 650 651 test_prctl(); 652 test_prctl_fork(); 653 test_prctl_fork_exec(); 654 test_prctl_unmerge(); 655 656 err = ksft_get_fail_cnt(); 657 if (err) 658 ksft_exit_fail_msg("%d out of %d tests failed\n", 659 err, ksft_test_num()); 660 ksft_exit_pass(); 661 } 662