1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright 2020 Google LLC 4 */ 5 #define _GNU_SOURCE 6 7 #include <errno.h> 8 #include <fcntl.h> 9 #include <linux/userfaultfd.h> 10 #include <stdlib.h> 11 #include <stdio.h> 12 #include <string.h> 13 #include <sys/ioctl.h> 14 #include <sys/mman.h> 15 #include <syscall.h> 16 #include <time.h> 17 #include <stdbool.h> 18 19 #include "kselftest.h" 20 21 #define EXPECT_SUCCESS 0 22 #define EXPECT_FAILURE 1 23 #define NON_OVERLAPPING 0 24 #define OVERLAPPING 1 25 #define NS_PER_SEC 1000000000ULL 26 #define VALIDATION_DEFAULT_THRESHOLD 4 /* 4MB */ 27 #define VALIDATION_NO_THRESHOLD 0 /* Verify the entire region */ 28 29 #ifndef MIN 30 #define MIN(X, Y) ((X) < (Y) ? (X) : (Y)) 31 #define MAX(X, Y) ((X) > (Y) ? (X) : (Y)) 32 #endif 33 #define SIZE_MB(m) ((size_t)m * (1024 * 1024)) 34 #define SIZE_KB(k) ((size_t)k * 1024) 35 36 struct config { 37 unsigned long long src_alignment; 38 unsigned long long dest_alignment; 39 unsigned long long region_size; 40 int overlapping; 41 unsigned int dest_preamble_size; 42 }; 43 44 struct test { 45 const char *name; 46 struct config config; 47 int expect_failure; 48 }; 49 50 enum { 51 _1KB = 1ULL << 10, /* 1KB -> not page aligned */ 52 _4KB = 4ULL << 10, 53 _8KB = 8ULL << 10, 54 _1MB = 1ULL << 20, 55 _2MB = 2ULL << 20, 56 _4MB = 4ULL << 20, 57 _5MB = 5ULL << 20, 58 _1GB = 1ULL << 30, 59 _2GB = 2ULL << 30, 60 PMD = _2MB, 61 PUD = _1GB, 62 }; 63 64 #define PTE page_size 65 66 #define MAKE_TEST(source_align, destination_align, size, \ 67 overlaps, should_fail, test_name) \ 68 (struct test){ \ 69 .name = test_name, \ 70 .config = { \ 71 .src_alignment = source_align, \ 72 .dest_alignment = destination_align, \ 73 .region_size = size, \ 74 .overlapping = overlaps, \ 75 }, \ 76 .expect_failure = should_fail \ 77 } 78 79 /* 80 * Returns false if the requested remap region overlaps with an 81 * existing mapping (e.g text, stack) else returns true. 82 */ 83 static bool is_remap_region_valid(void *addr, unsigned long long size) 84 { 85 void *remap_addr = NULL; 86 bool ret = true; 87 88 /* Use MAP_FIXED_NOREPLACE flag to ensure region is not mapped */ 89 remap_addr = mmap(addr, size, PROT_READ | PROT_WRITE, 90 MAP_FIXED_NOREPLACE | MAP_ANONYMOUS | MAP_SHARED, 91 -1, 0); 92 93 if (remap_addr == MAP_FAILED) { 94 if (errno == EEXIST) 95 ret = false; 96 } else { 97 munmap(remap_addr, size); 98 } 99 100 return ret; 101 } 102 103 /* Returns mmap_min_addr sysctl tunable from procfs */ 104 static unsigned long long get_mmap_min_addr(void) 105 { 106 FILE *fp; 107 int n_matched; 108 static unsigned long long addr; 109 110 if (addr) 111 return addr; 112 113 fp = fopen("/proc/sys/vm/mmap_min_addr", "r"); 114 if (fp == NULL) { 115 ksft_print_msg("Failed to open /proc/sys/vm/mmap_min_addr: %s\n", 116 strerror(errno)); 117 exit(KSFT_SKIP); 118 } 119 120 n_matched = fscanf(fp, "%llu", &addr); 121 if (n_matched != 1) { 122 ksft_print_msg("Failed to read /proc/sys/vm/mmap_min_addr: %s\n", 123 strerror(errno)); 124 fclose(fp); 125 exit(KSFT_SKIP); 126 } 127 128 fclose(fp); 129 return addr; 130 } 131 132 /* 133 * Using /proc/self/maps, assert that the specified address range is contained 134 * within a single mapping. 135 */ 136 static bool is_range_mapped(FILE *maps_fp, unsigned long start, 137 unsigned long end) 138 { 139 char *line = NULL; 140 size_t len = 0; 141 bool success = false; 142 unsigned long first_val, second_val; 143 144 rewind(maps_fp); 145 146 while (getline(&line, &len, maps_fp) != -1) { 147 if (sscanf(line, "%lx-%lx", &first_val, &second_val) != 2) { 148 ksft_exit_fail_msg("cannot parse /proc/self/maps\n"); 149 break; 150 } 151 152 if (first_val <= start && second_val >= end) { 153 success = true; 154 fflush(maps_fp); 155 break; 156 } 157 } 158 159 return success; 160 } 161 162 /* Check if [ptr, ptr + size) mapped in /proc/self/maps. */ 163 static bool is_ptr_mapped(FILE *maps_fp, void *ptr, unsigned long size) 164 { 165 unsigned long start = (unsigned long)ptr; 166 unsigned long end = start + size; 167 168 return is_range_mapped(maps_fp, start, end); 169 } 170 171 /* 172 * Returns the start address of the mapping on success, else returns 173 * NULL on failure. 174 */ 175 static void *get_source_mapping(struct config c) 176 { 177 unsigned long long addr = 0ULL; 178 void *src_addr = NULL; 179 unsigned long long mmap_min_addr; 180 181 mmap_min_addr = get_mmap_min_addr(); 182 /* 183 * For some tests, we need to not have any mappings below the 184 * source mapping. Add some headroom to mmap_min_addr for this. 185 */ 186 mmap_min_addr += 10 * _4MB; 187 188 retry: 189 addr += c.src_alignment; 190 if (addr < mmap_min_addr) 191 goto retry; 192 193 src_addr = mmap((void *) addr, c.region_size, PROT_READ | PROT_WRITE, 194 MAP_FIXED_NOREPLACE | MAP_ANONYMOUS | MAP_SHARED, 195 -1, 0); 196 if (src_addr == MAP_FAILED) { 197 if (errno == EPERM || errno == EEXIST) 198 goto retry; 199 goto error; 200 } 201 /* 202 * Check that the address is aligned to the specified alignment. 203 * Addresses which have alignments that are multiples of that 204 * specified are not considered valid. For instance, 1GB address is 205 * 2MB-aligned, however it will not be considered valid for a 206 * requested alignment of 2MB. This is done to reduce coincidental 207 * alignment in the tests. 208 */ 209 if (((unsigned long long) src_addr & (c.src_alignment - 1)) || 210 !((unsigned long long) src_addr & c.src_alignment)) { 211 munmap(src_addr, c.region_size); 212 goto retry; 213 } 214 215 if (!src_addr) 216 goto error; 217 218 return src_addr; 219 error: 220 ksft_print_msg("Failed to map source region: %s\n", 221 strerror(errno)); 222 return NULL; 223 } 224 225 /* 226 * This test validates that merge is called when expanding a mapping. 227 * Mapping containing three pages is created, middle page is unmapped 228 * and then the mapping containing the first page is expanded so that 229 * it fills the created hole. The two parts should merge creating 230 * single mapping with three pages. 231 */ 232 static void mremap_expand_merge(FILE *maps_fp, unsigned long page_size) 233 { 234 char *test_name = "mremap expand merge"; 235 bool success = false; 236 char *remap, *start; 237 238 start = mmap(NULL, 3 * page_size, PROT_READ | PROT_WRITE, 239 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); 240 241 if (start == MAP_FAILED) { 242 ksft_print_msg("mmap failed: %s\n", strerror(errno)); 243 goto out; 244 } 245 246 munmap(start + page_size, page_size); 247 remap = mremap(start, page_size, 2 * page_size, 0); 248 if (remap == MAP_FAILED) { 249 ksft_print_msg("mremap failed: %s\n", strerror(errno)); 250 munmap(start, page_size); 251 munmap(start + 2 * page_size, page_size); 252 goto out; 253 } 254 255 success = is_range_mapped(maps_fp, (unsigned long)start, 256 (unsigned long)(start + 3 * page_size)); 257 munmap(start, 3 * page_size); 258 259 out: 260 if (success) 261 ksft_test_result_pass("%s\n", test_name); 262 else 263 ksft_test_result_fail("%s\n", test_name); 264 } 265 266 /* 267 * Similar to mremap_expand_merge() except instead of removing the middle page, 268 * we remove the last then attempt to remap offset from the second page. This 269 * should result in the mapping being restored to its former state. 270 */ 271 static void mremap_expand_merge_offset(FILE *maps_fp, unsigned long page_size) 272 { 273 274 char *test_name = "mremap expand merge offset"; 275 bool success = false; 276 char *remap, *start; 277 278 start = mmap(NULL, 3 * page_size, PROT_READ | PROT_WRITE, 279 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); 280 281 if (start == MAP_FAILED) { 282 ksft_print_msg("mmap failed: %s\n", strerror(errno)); 283 goto out; 284 } 285 286 /* Unmap final page to ensure we have space to expand. */ 287 munmap(start + 2 * page_size, page_size); 288 remap = mremap(start + page_size, page_size, 2 * page_size, 0); 289 if (remap == MAP_FAILED) { 290 ksft_print_msg("mremap failed: %s\n", strerror(errno)); 291 munmap(start, 2 * page_size); 292 goto out; 293 } 294 295 success = is_range_mapped(maps_fp, (unsigned long)start, 296 (unsigned long)(start + 3 * page_size)); 297 munmap(start, 3 * page_size); 298 299 out: 300 if (success) 301 ksft_test_result_pass("%s\n", test_name); 302 else 303 ksft_test_result_fail("%s\n", test_name); 304 } 305 306 /* 307 * Verify that an mremap within a range does not cause corruption 308 * of unrelated part of range. 309 * 310 * Consider the following range which is 2MB aligned and is 311 * a part of a larger 20MB range which is not shown. Each 312 * character is 256KB below making the source and destination 313 * 2MB each. The lower case letters are moved (s to d) and the 314 * upper case letters are not moved. The below test verifies 315 * that the upper case S letters are not corrupted by the 316 * adjacent mremap. 317 * 318 * |DDDDddddSSSSssss| 319 */ 320 static void mremap_move_within_range(unsigned int pattern_seed, char *rand_addr) 321 { 322 char *test_name = "mremap mremap move within range"; 323 void *src, *dest; 324 unsigned int i, success = 1; 325 326 size_t size = SIZE_MB(20); 327 void *ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, 328 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); 329 if (ptr == MAP_FAILED) { 330 perror("mmap"); 331 success = 0; 332 goto out; 333 } 334 memset(ptr, 0, size); 335 336 src = ptr + SIZE_MB(6); 337 src = (void *)((unsigned long)src & ~(SIZE_MB(2) - 1)); 338 339 /* Set byte pattern for source block. */ 340 memcpy(src, rand_addr, SIZE_MB(2)); 341 342 dest = src - SIZE_MB(2); 343 344 void *new_ptr = mremap(src + SIZE_MB(1), SIZE_MB(1), SIZE_MB(1), 345 MREMAP_MAYMOVE | MREMAP_FIXED, dest + SIZE_MB(1)); 346 if (new_ptr == MAP_FAILED) { 347 perror("mremap"); 348 success = 0; 349 goto out; 350 } 351 352 /* Verify byte pattern after remapping */ 353 srand(pattern_seed); 354 for (i = 0; i < SIZE_MB(1); i++) { 355 char c = (char) rand(); 356 357 if (((char *)src)[i] != c) { 358 ksft_print_msg("Data at src at %d got corrupted due to unrelated mremap\n", 359 i); 360 ksft_print_msg("Expected: %#x\t Got: %#x\n", c & 0xff, 361 ((char *) src)[i] & 0xff); 362 success = 0; 363 } 364 } 365 366 out: 367 if (munmap(ptr, size) == -1) 368 perror("munmap"); 369 370 if (success) 371 ksft_test_result_pass("%s\n", test_name); 372 else 373 ksft_test_result_fail("%s\n", test_name); 374 } 375 376 static bool is_multiple_vma_range_ok(unsigned int pattern_seed, 377 char *ptr, unsigned long page_size) 378 { 379 int i; 380 381 srand(pattern_seed); 382 for (i = 0; i <= 10; i += 2) { 383 int j; 384 char *buf = &ptr[i * page_size]; 385 size_t size = i == 4 ? 2 * page_size : page_size; 386 387 for (j = 0; j < size; j++) { 388 char chr = rand(); 389 390 if (chr != buf[j]) { 391 ksft_print_msg("page %d offset %d corrupted, expected %d got %d\n", 392 i, j, chr, buf[j]); 393 return false; 394 } 395 } 396 } 397 398 return true; 399 } 400 401 static void mremap_move_multiple_vmas(unsigned int pattern_seed, 402 unsigned long page_size, 403 bool dont_unmap) 404 { 405 int mremap_flags = MREMAP_FIXED | MREMAP_MAYMOVE; 406 char *test_name = "mremap move multiple vmas"; 407 const size_t size = 11 * page_size; 408 bool success = true; 409 char *ptr, *tgt_ptr; 410 int i; 411 412 if (dont_unmap) 413 mremap_flags |= MREMAP_DONTUNMAP; 414 415 ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, 416 MAP_PRIVATE | MAP_ANON, -1, 0); 417 if (ptr == MAP_FAILED) { 418 perror("mmap"); 419 success = false; 420 goto out; 421 } 422 423 tgt_ptr = mmap(NULL, 2 * size, PROT_READ | PROT_WRITE, 424 MAP_PRIVATE | MAP_ANON, -1, 0); 425 if (tgt_ptr == MAP_FAILED) { 426 perror("mmap"); 427 success = false; 428 goto out; 429 } 430 if (munmap(tgt_ptr, 2 * size)) { 431 perror("munmap"); 432 success = false; 433 goto out_unmap; 434 } 435 436 /* 437 * Unmap so we end up with: 438 * 439 * 0 2 4 5 6 8 10 offset in buffer 440 * |*| |*| |*****| |*| |*| 441 * |*| |*| |*****| |*| |*| 442 * 0 1 2 3 4 5 6 pattern offset 443 */ 444 for (i = 1; i < 10; i += 2) { 445 if (i == 5) 446 continue; 447 448 if (munmap(&ptr[i * page_size], page_size)) { 449 perror("munmap"); 450 success = false; 451 goto out_unmap; 452 } 453 } 454 455 srand(pattern_seed); 456 457 /* Set up random patterns. */ 458 for (i = 0; i <= 10; i += 2) { 459 int j; 460 size_t size = i == 4 ? 2 * page_size : page_size; 461 char *buf = &ptr[i * page_size]; 462 463 for (j = 0; j < size; j++) 464 buf[j] = rand(); 465 } 466 467 /* First, just move the whole thing. */ 468 if (mremap(ptr, size, size, mremap_flags, tgt_ptr) == MAP_FAILED) { 469 perror("mremap"); 470 success = false; 471 goto out_unmap; 472 } 473 /* Check move was ok. */ 474 if (!is_multiple_vma_range_ok(pattern_seed, tgt_ptr, page_size)) { 475 success = false; 476 goto out_unmap; 477 } 478 479 /* Move next to itself. */ 480 if (mremap(tgt_ptr, size, size, mremap_flags, 481 &tgt_ptr[size]) == MAP_FAILED) { 482 perror("mremap"); 483 success = false; 484 goto out_unmap; 485 } 486 /* Check that the move is ok. */ 487 if (!is_multiple_vma_range_ok(pattern_seed, &tgt_ptr[size], page_size)) { 488 success = false; 489 goto out_unmap; 490 } 491 492 /* Map a range to overwrite. */ 493 if (mmap(tgt_ptr, size, PROT_NONE, 494 MAP_PRIVATE | MAP_ANON | MAP_FIXED, -1, 0) == MAP_FAILED) { 495 perror("mmap tgt"); 496 success = false; 497 goto out_unmap; 498 } 499 /* Move and overwrite. */ 500 if (mremap(&tgt_ptr[size], size, size, 501 mremap_flags, tgt_ptr) == MAP_FAILED) { 502 perror("mremap"); 503 success = false; 504 goto out_unmap; 505 } 506 /* Check that the move is ok. */ 507 if (!is_multiple_vma_range_ok(pattern_seed, tgt_ptr, page_size)) { 508 success = false; 509 goto out_unmap; 510 } 511 512 out_unmap: 513 if (munmap(tgt_ptr, 2 * size)) 514 perror("munmap tgt"); 515 if (munmap(ptr, size)) 516 perror("munmap src"); 517 518 out: 519 if (success) 520 ksft_test_result_pass("%s%s\n", test_name, 521 dont_unmap ? " [dontunnmap]" : ""); 522 else 523 ksft_test_result_fail("%s%s\n", test_name, 524 dont_unmap ? " [dontunnmap]" : ""); 525 } 526 527 static void mremap_shrink_multiple_vmas(unsigned long page_size, 528 bool inplace) 529 { 530 char *test_name = "mremap shrink multiple vmas"; 531 const size_t size = 10 * page_size; 532 bool success = true; 533 char *ptr, *tgt_ptr; 534 void *res; 535 int i; 536 537 ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, 538 MAP_PRIVATE | MAP_ANON, -1, 0); 539 if (ptr == MAP_FAILED) { 540 perror("mmap"); 541 success = false; 542 goto out; 543 } 544 545 tgt_ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, 546 MAP_PRIVATE | MAP_ANON, -1, 0); 547 if (tgt_ptr == MAP_FAILED) { 548 perror("mmap"); 549 success = false; 550 goto out; 551 } 552 if (munmap(tgt_ptr, size)) { 553 perror("munmap"); 554 success = false; 555 goto out_unmap; 556 } 557 558 /* 559 * Unmap so we end up with: 560 * 561 * 0 2 4 6 8 10 offset in buffer 562 * |*| |*| |*| |*| |*| |*| 563 * |*| |*| |*| |*| |*| |*| 564 */ 565 for (i = 1; i < 10; i += 2) { 566 if (munmap(&ptr[i * page_size], page_size)) { 567 perror("munmap"); 568 success = false; 569 goto out_unmap; 570 } 571 } 572 573 /* 574 * Shrink in-place across multiple VMAs and gaps so we end up with: 575 * 576 * 0 577 * |*| 578 * |*| 579 */ 580 if (inplace) 581 res = mremap(ptr, size, page_size, 0); 582 else 583 res = mremap(ptr, size, page_size, MREMAP_MAYMOVE | MREMAP_FIXED, 584 tgt_ptr); 585 586 if (res == MAP_FAILED) { 587 perror("mremap"); 588 success = false; 589 goto out_unmap; 590 } 591 592 out_unmap: 593 if (munmap(tgt_ptr, size)) 594 perror("munmap tgt"); 595 if (munmap(ptr, size)) 596 perror("munmap src"); 597 out: 598 if (success) 599 ksft_test_result_pass("%s%s\n", test_name, 600 inplace ? " [inplace]" : ""); 601 else 602 ksft_test_result_fail("%s%s\n", test_name, 603 inplace ? " [inplace]" : ""); 604 } 605 606 static void mremap_move_multiple_vmas_split(unsigned int pattern_seed, 607 unsigned long page_size, 608 bool dont_unmap) 609 { 610 char *test_name = "mremap move multiple vmas split"; 611 int mremap_flags = MREMAP_FIXED | MREMAP_MAYMOVE; 612 const size_t size = 10 * page_size; 613 bool success = true; 614 char *ptr, *tgt_ptr; 615 int i; 616 617 if (dont_unmap) 618 mremap_flags |= MREMAP_DONTUNMAP; 619 620 ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, 621 MAP_PRIVATE | MAP_ANON, -1, 0); 622 if (ptr == MAP_FAILED) { 623 perror("mmap"); 624 success = false; 625 goto out; 626 } 627 628 tgt_ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, 629 MAP_PRIVATE | MAP_ANON, -1, 0); 630 if (tgt_ptr == MAP_FAILED) { 631 perror("mmap"); 632 success = false; 633 goto out; 634 } 635 if (munmap(tgt_ptr, size)) { 636 perror("munmap"); 637 success = false; 638 goto out_unmap; 639 } 640 641 /* 642 * Unmap so we end up with: 643 * 644 * 0 1 2 3 4 5 6 7 8 9 10 offset in buffer 645 * |**********| |*******| 646 * |**********| |*******| 647 * 0 1 2 3 4 5 6 7 8 9 pattern offset 648 */ 649 if (munmap(&ptr[5 * page_size], page_size)) { 650 perror("munmap"); 651 success = false; 652 goto out_unmap; 653 } 654 655 /* Set up random patterns. */ 656 srand(pattern_seed); 657 for (i = 0; i < 10; i++) { 658 int j; 659 char *buf = &ptr[i * page_size]; 660 661 if (i == 5) 662 continue; 663 664 for (j = 0; j < page_size; j++) 665 buf[j] = rand(); 666 } 667 668 /* 669 * Move the below: 670 * 671 * <-------------> 672 * 0 1 2 3 4 5 6 7 8 9 10 offset in buffer 673 * |**********| |*******| 674 * |**********| |*******| 675 * 0 1 2 3 4 5 6 7 8 9 pattern offset 676 * 677 * Into: 678 * 679 * 0 1 2 3 4 5 6 7 offset in buffer 680 * |*****| |*****| 681 * |*****| |*****| 682 * 2 3 4 5 6 7 pattern offset 683 */ 684 if (mremap(&ptr[2 * page_size], size - 3 * page_size, size - 3 * page_size, 685 mremap_flags, tgt_ptr) == MAP_FAILED) { 686 perror("mremap"); 687 success = false; 688 goto out_unmap; 689 } 690 691 /* Offset into random pattern. */ 692 srand(pattern_seed); 693 for (i = 0; i < 2 * page_size; i++) 694 rand(); 695 696 /* Check pattern. */ 697 for (i = 0; i < 7; i++) { 698 int j; 699 char *buf = &tgt_ptr[i * page_size]; 700 701 if (i == 3) 702 continue; 703 704 for (j = 0; j < page_size; j++) { 705 char chr = rand(); 706 707 if (chr != buf[j]) { 708 ksft_print_msg("page %d offset %d corrupted, expected %d got %d\n", 709 i, j, chr, buf[j]); 710 goto out_unmap; 711 } 712 } 713 } 714 715 out_unmap: 716 if (munmap(tgt_ptr, size)) 717 perror("munmap tgt"); 718 if (munmap(ptr, size)) 719 perror("munmap src"); 720 out: 721 if (success) 722 ksft_test_result_pass("%s%s\n", test_name, 723 dont_unmap ? " [dontunnmap]" : ""); 724 else 725 ksft_test_result_fail("%s%s\n", test_name, 726 dont_unmap ? " [dontunnmap]" : ""); 727 } 728 729 #ifdef __NR_userfaultfd 730 static void mremap_move_multi_invalid_vmas(FILE *maps_fp, 731 unsigned long page_size) 732 { 733 char *test_name = "mremap move multiple invalid vmas"; 734 const size_t size = 10 * page_size; 735 bool success = true; 736 char *ptr, *tgt_ptr; 737 int uffd, err, i; 738 void *res; 739 struct uffdio_api api = { 740 .api = UFFD_API, 741 .features = UFFD_EVENT_PAGEFAULT, 742 }; 743 744 uffd = syscall(__NR_userfaultfd, O_NONBLOCK); 745 if (uffd == -1) { 746 err = errno; 747 perror("userfaultfd"); 748 if (err == EPERM) { 749 ksft_test_result_skip("%s - missing uffd", test_name); 750 return; 751 } 752 success = false; 753 goto out; 754 } 755 if (ioctl(uffd, UFFDIO_API, &api)) { 756 perror("ioctl UFFDIO_API"); 757 success = false; 758 goto out_close_uffd; 759 } 760 761 ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, 762 MAP_PRIVATE | MAP_ANON, -1, 0); 763 if (ptr == MAP_FAILED) { 764 perror("mmap"); 765 success = false; 766 goto out_close_uffd; 767 } 768 769 tgt_ptr = mmap(NULL, size, PROT_NONE, MAP_PRIVATE | MAP_ANON, -1, 0); 770 if (tgt_ptr == MAP_FAILED) { 771 perror("mmap"); 772 success = false; 773 goto out_close_uffd; 774 } 775 if (munmap(tgt_ptr, size)) { 776 perror("munmap"); 777 success = false; 778 goto out_unmap; 779 } 780 781 /* 782 * Unmap so we end up with: 783 * 784 * 0 2 4 6 8 10 offset in buffer 785 * |*| |*| |*| |*| |*| 786 * |*| |*| |*| |*| |*| 787 * 788 * Additionally, register each with UFFD. 789 */ 790 for (i = 0; i < 10; i += 2) { 791 void *unmap_ptr = &ptr[(i + 1) * page_size]; 792 unsigned long start = (unsigned long)&ptr[i * page_size]; 793 struct uffdio_register reg = { 794 .range = { 795 .start = start, 796 .len = page_size, 797 }, 798 .mode = UFFDIO_REGISTER_MODE_MISSING, 799 }; 800 801 if (ioctl(uffd, UFFDIO_REGISTER, ®) == -1) { 802 perror("ioctl UFFDIO_REGISTER"); 803 success = false; 804 goto out_unmap; 805 } 806 if (munmap(unmap_ptr, page_size)) { 807 perror("munmap"); 808 success = false; 809 goto out_unmap; 810 } 811 } 812 813 /* 814 * Now try to move the entire range which is invalid for multi VMA move. 815 * 816 * This will fail, and no VMA should be moved, as we check this ahead of 817 * time. 818 */ 819 res = mremap(ptr, size, size, MREMAP_MAYMOVE | MREMAP_FIXED, tgt_ptr); 820 err = errno; 821 if (res != MAP_FAILED) { 822 fprintf(stderr, "mremap() succeeded for multi VMA uffd armed\n"); 823 success = false; 824 goto out_unmap; 825 } 826 if (err != EFAULT) { 827 errno = err; 828 perror("mremap() unexpected error"); 829 success = false; 830 goto out_unmap; 831 } 832 if (is_ptr_mapped(maps_fp, tgt_ptr, page_size)) { 833 fprintf(stderr, 834 "Invalid uffd-armed VMA at start of multi range moved\n"); 835 success = false; 836 goto out_unmap; 837 } 838 839 /* 840 * Now try to move a single VMA, this should succeed as not multi VMA 841 * move. 842 */ 843 res = mremap(ptr, page_size, page_size, 844 MREMAP_MAYMOVE | MREMAP_FIXED, tgt_ptr); 845 if (res == MAP_FAILED) { 846 perror("mremap single invalid-multi VMA"); 847 success = false; 848 goto out_unmap; 849 } 850 851 /* 852 * Unmap the VMA, and remap a non-uffd registered (therefore, multi VMA 853 * move valid) VMA at the start of ptr range. 854 */ 855 if (munmap(tgt_ptr, page_size)) { 856 perror("munmap"); 857 success = false; 858 goto out_unmap; 859 } 860 res = mmap(ptr, page_size, PROT_READ | PROT_WRITE, 861 MAP_PRIVATE | MAP_ANON | MAP_FIXED, -1, 0); 862 if (res == MAP_FAILED) { 863 perror("mmap"); 864 success = false; 865 goto out_unmap; 866 } 867 868 /* 869 * Now try to move the entire range, we should succeed in moving the 870 * first VMA, but no others, and report a failure. 871 */ 872 res = mremap(ptr, size, size, MREMAP_MAYMOVE | MREMAP_FIXED, tgt_ptr); 873 err = errno; 874 if (res != MAP_FAILED) { 875 fprintf(stderr, "mremap() succeeded for multi VMA uffd armed\n"); 876 success = false; 877 goto out_unmap; 878 } 879 if (err != EFAULT) { 880 errno = err; 881 perror("mremap() unexpected error"); 882 success = false; 883 goto out_unmap; 884 } 885 if (!is_ptr_mapped(maps_fp, tgt_ptr, page_size)) { 886 fprintf(stderr, "Valid VMA not moved\n"); 887 success = false; 888 goto out_unmap; 889 } 890 891 /* 892 * Unmap the VMA, and map valid VMA at start of ptr range, and replace 893 * all existing multi-move invalid VMAs, except the last, with valid 894 * multi-move VMAs. 895 */ 896 if (munmap(tgt_ptr, page_size)) { 897 perror("munmap"); 898 success = false; 899 goto out_unmap; 900 } 901 if (munmap(ptr, size - 2 * page_size)) { 902 perror("munmap"); 903 success = false; 904 goto out_unmap; 905 } 906 for (i = 0; i < 8; i += 2) { 907 res = mmap(&ptr[i * page_size], page_size, 908 PROT_READ | PROT_WRITE, 909 MAP_PRIVATE | MAP_ANON | MAP_FIXED, -1, 0); 910 if (res == MAP_FAILED) { 911 perror("mmap"); 912 success = false; 913 goto out_unmap; 914 } 915 } 916 917 /* 918 * Now try to move the entire range, we should succeed in moving all but 919 * the last VMA, and report a failure. 920 */ 921 res = mremap(ptr, size, size, MREMAP_MAYMOVE | MREMAP_FIXED, tgt_ptr); 922 err = errno; 923 if (res != MAP_FAILED) { 924 fprintf(stderr, "mremap() succeeded for multi VMA uffd armed\n"); 925 success = false; 926 goto out_unmap; 927 } 928 if (err != EFAULT) { 929 errno = err; 930 perror("mremap() unexpected error"); 931 success = false; 932 goto out_unmap; 933 } 934 935 for (i = 0; i < 10; i += 2) { 936 bool is_mapped = is_ptr_mapped(maps_fp, 937 &tgt_ptr[i * page_size], page_size); 938 939 if (i < 8 && !is_mapped) { 940 fprintf(stderr, "Valid VMA not moved at %d\n", i); 941 success = false; 942 goto out_unmap; 943 } else if (i == 8 && is_mapped) { 944 fprintf(stderr, "Invalid VMA moved at %d\n", i); 945 success = false; 946 goto out_unmap; 947 } 948 } 949 950 out_unmap: 951 if (munmap(tgt_ptr, size)) 952 perror("munmap tgt"); 953 if (munmap(ptr, size)) 954 perror("munmap src"); 955 out_close_uffd: 956 close(uffd); 957 out: 958 if (success) 959 ksft_test_result_pass("%s\n", test_name); 960 else 961 ksft_test_result_fail("%s\n", test_name); 962 } 963 #else 964 static void mremap_move_multi_invalid_vmas(FILE *maps_fp, unsigned long page_size) 965 { 966 char *test_name = "mremap move multiple invalid vmas"; 967 968 ksft_test_result_skip("%s - missing uffd", test_name); 969 } 970 #endif /* __NR_userfaultfd */ 971 972 /* Returns the time taken for the remap on success else returns -1. */ 973 static long long remap_region(struct config c, unsigned int threshold_mb, 974 char *rand_addr) 975 { 976 void *addr, *tmp_addr, *src_addr, *dest_addr, *dest_preamble_addr = NULL; 977 struct timespec t_start = {0, 0}, t_end = {0, 0}; 978 long long start_ns, end_ns, align_mask, ret, offset; 979 unsigned long long threshold; 980 981 if (threshold_mb == VALIDATION_NO_THRESHOLD) 982 threshold = c.region_size; 983 else 984 threshold = MIN(threshold_mb * _1MB, c.region_size); 985 986 src_addr = get_source_mapping(c); 987 if (!src_addr) { 988 ret = -1; 989 goto out; 990 } 991 992 /* Set byte pattern for source block. */ 993 memcpy(src_addr, rand_addr, threshold); 994 995 /* Mask to zero out lower bits of address for alignment */ 996 align_mask = ~(c.dest_alignment - 1); 997 /* Offset of destination address from the end of the source region */ 998 offset = (c.overlapping) ? -c.dest_alignment : c.dest_alignment; 999 addr = (void *) (((unsigned long long) src_addr + c.region_size 1000 + offset) & align_mask); 1001 1002 /* Remap after the destination block preamble. */ 1003 addr += c.dest_preamble_size; 1004 1005 /* See comment in get_source_mapping() */ 1006 if (!((unsigned long long) addr & c.dest_alignment)) 1007 addr = (void *) ((unsigned long long) addr | c.dest_alignment); 1008 1009 /* Don't destroy existing mappings unless expected to overlap */ 1010 while (!is_remap_region_valid(addr, c.region_size) && !c.overlapping) { 1011 /* Check for unsigned overflow */ 1012 tmp_addr = addr + c.dest_alignment; 1013 if (tmp_addr < addr) { 1014 ksft_print_msg("Couldn't find a valid region to remap to\n"); 1015 ret = -1; 1016 goto clean_up_src; 1017 } 1018 addr += c.dest_alignment; 1019 } 1020 1021 if (c.dest_preamble_size) { 1022 dest_preamble_addr = mmap((void *) addr - c.dest_preamble_size, c.dest_preamble_size, 1023 PROT_READ | PROT_WRITE, 1024 MAP_FIXED_NOREPLACE | MAP_ANONYMOUS | MAP_SHARED, 1025 -1, 0); 1026 if (dest_preamble_addr == MAP_FAILED) { 1027 ksft_print_msg("Failed to map dest preamble region: %s\n", 1028 strerror(errno)); 1029 ret = -1; 1030 goto clean_up_src; 1031 } 1032 1033 /* Set byte pattern for the dest preamble block. */ 1034 memcpy(dest_preamble_addr, rand_addr, c.dest_preamble_size); 1035 } 1036 1037 clock_gettime(CLOCK_MONOTONIC, &t_start); 1038 dest_addr = mremap(src_addr, c.region_size, c.region_size, 1039 MREMAP_MAYMOVE|MREMAP_FIXED, (char *) addr); 1040 clock_gettime(CLOCK_MONOTONIC, &t_end); 1041 1042 if (dest_addr == MAP_FAILED) { 1043 ksft_print_msg("mremap failed: %s\n", strerror(errno)); 1044 ret = -1; 1045 goto clean_up_dest_preamble; 1046 } 1047 1048 /* Verify byte pattern after remapping */ 1049 if (memcmp(dest_addr, rand_addr, threshold)) { 1050 ksft_print_msg("Data after remap doesn't match\n"); 1051 ret = -1; 1052 goto clean_up_dest; 1053 } 1054 1055 /* Verify the dest preamble byte pattern after remapping */ 1056 if (c.dest_preamble_size && 1057 memcmp(dest_preamble_addr, rand_addr, c.dest_preamble_size)) { 1058 ksft_print_msg("Preamble data after remap doesn't match\n"); 1059 ret = -1; 1060 goto clean_up_dest; 1061 } 1062 1063 start_ns = t_start.tv_sec * NS_PER_SEC + t_start.tv_nsec; 1064 end_ns = t_end.tv_sec * NS_PER_SEC + t_end.tv_nsec; 1065 ret = end_ns - start_ns; 1066 1067 /* 1068 * Since the destination address is specified using MREMAP_FIXED, subsequent 1069 * mremap will unmap any previous mapping at the address range specified by 1070 * dest_addr and region_size. This significantly affects the remap time of 1071 * subsequent tests. So we clean up mappings after each test. 1072 */ 1073 clean_up_dest: 1074 munmap(dest_addr, c.region_size); 1075 clean_up_dest_preamble: 1076 if (c.dest_preamble_size && dest_preamble_addr) 1077 munmap(dest_preamble_addr, c.dest_preamble_size); 1078 clean_up_src: 1079 munmap(src_addr, c.region_size); 1080 out: 1081 return ret; 1082 } 1083 1084 /* 1085 * Verify that an mremap aligning down does not destroy 1086 * the beginning of the mapping just because the aligned 1087 * down address landed on a mapping that maybe does not exist. 1088 */ 1089 static void mremap_move_1mb_from_start(unsigned int pattern_seed, 1090 char *rand_addr) 1091 { 1092 char *test_name = "mremap move 1mb from start at 1MB+256KB aligned src"; 1093 void *src = NULL, *dest = NULL; 1094 unsigned int i, success = 1; 1095 1096 /* Config to reuse get_source_mapping() to do an aligned mmap. */ 1097 struct config c = { 1098 .src_alignment = SIZE_MB(1) + SIZE_KB(256), 1099 .region_size = SIZE_MB(6) 1100 }; 1101 1102 src = get_source_mapping(c); 1103 if (!src) { 1104 success = 0; 1105 goto out; 1106 } 1107 1108 c.src_alignment = SIZE_MB(1) + SIZE_KB(256); 1109 dest = get_source_mapping(c); 1110 if (!dest) { 1111 success = 0; 1112 goto out; 1113 } 1114 1115 /* Set byte pattern for source block. */ 1116 memcpy(src, rand_addr, SIZE_MB(2)); 1117 1118 /* 1119 * Unmap the beginning of dest so that the aligned address 1120 * falls on no mapping. 1121 */ 1122 munmap(dest, SIZE_MB(1)); 1123 1124 void *new_ptr = mremap(src + SIZE_MB(1), SIZE_MB(1), SIZE_MB(1), 1125 MREMAP_MAYMOVE | MREMAP_FIXED, dest + SIZE_MB(1)); 1126 if (new_ptr == MAP_FAILED) { 1127 perror("mremap"); 1128 success = 0; 1129 goto out; 1130 } 1131 1132 /* Verify byte pattern after remapping */ 1133 srand(pattern_seed); 1134 for (i = 0; i < SIZE_MB(1); i++) { 1135 char c = (char) rand(); 1136 1137 if (((char *)src)[i] != c) { 1138 ksft_print_msg("Data at src at %d got corrupted due to unrelated mremap\n", 1139 i); 1140 ksft_print_msg("Expected: %#x\t Got: %#x\n", c & 0xff, 1141 ((char *) src)[i] & 0xff); 1142 success = 0; 1143 } 1144 } 1145 1146 out: 1147 if (src && munmap(src, c.region_size) == -1) 1148 perror("munmap src"); 1149 1150 if (dest && munmap(dest, c.region_size) == -1) 1151 perror("munmap dest"); 1152 1153 if (success) 1154 ksft_test_result_pass("%s\n", test_name); 1155 else 1156 ksft_test_result_fail("%s\n", test_name); 1157 } 1158 1159 static void run_mremap_test_case(struct test test_case, int *failures, 1160 unsigned int threshold_mb, 1161 char *rand_addr) 1162 { 1163 long long remap_time = remap_region(test_case.config, threshold_mb, 1164 rand_addr); 1165 1166 if (remap_time < 0) { 1167 if (test_case.expect_failure) 1168 ksft_test_result_xfail("%s\n\tExpected mremap failure\n", 1169 test_case.name); 1170 else { 1171 ksft_test_result_fail("%s\n", test_case.name); 1172 *failures += 1; 1173 } 1174 } else { 1175 /* 1176 * Comparing mremap time is only applicable if entire region 1177 * was faulted in. 1178 */ 1179 if (threshold_mb == VALIDATION_NO_THRESHOLD || 1180 test_case.config.region_size <= threshold_mb * _1MB) 1181 ksft_test_result_pass("%s\n\tmremap time: %12lldns\n", 1182 test_case.name, remap_time); 1183 else 1184 ksft_test_result_pass("%s\n", test_case.name); 1185 } 1186 } 1187 1188 static void usage(const char *cmd) 1189 { 1190 fprintf(stderr, 1191 "Usage: %s [[-t <threshold_mb>] [-p <pattern_seed>]]\n" 1192 "-t\t only validate threshold_mb of the remapped region\n" 1193 " \t if 0 is supplied no threshold is used; all tests\n" 1194 " \t are run and remapped regions validated fully.\n" 1195 " \t The default threshold used is 4MB.\n" 1196 "-p\t provide a seed to generate the random pattern for\n" 1197 " \t validating the remapped region.\n", cmd); 1198 } 1199 1200 static int parse_args(int argc, char **argv, unsigned int *threshold_mb, 1201 unsigned int *pattern_seed) 1202 { 1203 const char *optstr = "t:p:"; 1204 int opt; 1205 1206 while ((opt = getopt(argc, argv, optstr)) != -1) { 1207 switch (opt) { 1208 case 't': 1209 *threshold_mb = atoi(optarg); 1210 break; 1211 case 'p': 1212 *pattern_seed = atoi(optarg); 1213 break; 1214 default: 1215 usage(argv[0]); 1216 return -1; 1217 } 1218 } 1219 1220 if (optind < argc) { 1221 usage(argv[0]); 1222 return -1; 1223 } 1224 1225 return 0; 1226 } 1227 1228 #define MAX_TEST 15 1229 #define MAX_PERF_TEST 3 1230 int main(int argc, char **argv) 1231 { 1232 int failures = 0; 1233 unsigned int i; 1234 int run_perf_tests; 1235 unsigned int threshold_mb = VALIDATION_DEFAULT_THRESHOLD; 1236 1237 /* hard-coded test configs */ 1238 size_t max_test_variable_region_size = _2GB; 1239 size_t max_test_constant_region_size = _2MB; 1240 size_t dest_preamble_size = 10 * _4MB; 1241 1242 unsigned int pattern_seed; 1243 char *rand_addr; 1244 size_t rand_size; 1245 int num_expand_tests = 2; 1246 int num_misc_tests = 9; 1247 struct test test_cases[MAX_TEST] = {}; 1248 struct test perf_test_cases[MAX_PERF_TEST]; 1249 int page_size; 1250 time_t t; 1251 FILE *maps_fp; 1252 1253 pattern_seed = (unsigned int) time(&t); 1254 1255 if (parse_args(argc, argv, &threshold_mb, &pattern_seed) < 0) 1256 exit(EXIT_FAILURE); 1257 1258 ksft_print_msg("Test configs:\n\tthreshold_mb=%u\n\tpattern_seed=%u\n\n", 1259 threshold_mb, pattern_seed); 1260 1261 /* 1262 * set preallocated random array according to test configs; see the 1263 * functions for the logic of setting the size 1264 */ 1265 if (!threshold_mb) 1266 rand_size = MAX(max_test_variable_region_size, 1267 max_test_constant_region_size); 1268 else 1269 rand_size = MAX(MIN(threshold_mb * _1MB, 1270 max_test_variable_region_size), 1271 max_test_constant_region_size); 1272 rand_size = MAX(dest_preamble_size, rand_size); 1273 1274 rand_addr = (char *)mmap(NULL, rand_size, PROT_READ | PROT_WRITE, 1275 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); 1276 if (rand_addr == MAP_FAILED) { 1277 perror("mmap"); 1278 ksft_exit_fail_msg("cannot mmap rand_addr\n"); 1279 } 1280 1281 /* fill stream of random bytes */ 1282 srand(pattern_seed); 1283 for (unsigned long i = 0; i < rand_size; ++i) 1284 rand_addr[i] = (char) rand(); 1285 1286 page_size = sysconf(_SC_PAGESIZE); 1287 1288 /* Expected mremap failures */ 1289 test_cases[0] = MAKE_TEST(page_size, page_size, page_size, 1290 OVERLAPPING, EXPECT_FAILURE, 1291 "mremap - Source and Destination Regions Overlapping"); 1292 1293 test_cases[1] = MAKE_TEST(page_size, page_size/4, page_size, 1294 NON_OVERLAPPING, EXPECT_FAILURE, 1295 "mremap - Destination Address Misaligned (1KB-aligned)"); 1296 test_cases[2] = MAKE_TEST(page_size/4, page_size, page_size, 1297 NON_OVERLAPPING, EXPECT_FAILURE, 1298 "mremap - Source Address Misaligned (1KB-aligned)"); 1299 1300 /* Src addr PTE aligned */ 1301 test_cases[3] = MAKE_TEST(PTE, PTE, PTE * 2, 1302 NON_OVERLAPPING, EXPECT_SUCCESS, 1303 "8KB mremap - Source PTE-aligned, Destination PTE-aligned"); 1304 1305 /* Src addr 1MB aligned */ 1306 test_cases[4] = MAKE_TEST(_1MB, PTE, _2MB, NON_OVERLAPPING, EXPECT_SUCCESS, 1307 "2MB mremap - Source 1MB-aligned, Destination PTE-aligned"); 1308 test_cases[5] = MAKE_TEST(_1MB, _1MB, _2MB, NON_OVERLAPPING, EXPECT_SUCCESS, 1309 "2MB mremap - Source 1MB-aligned, Destination 1MB-aligned"); 1310 1311 /* Src addr PMD aligned */ 1312 test_cases[6] = MAKE_TEST(PMD, PTE, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS, 1313 "4MB mremap - Source PMD-aligned, Destination PTE-aligned"); 1314 test_cases[7] = MAKE_TEST(PMD, _1MB, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS, 1315 "4MB mremap - Source PMD-aligned, Destination 1MB-aligned"); 1316 test_cases[8] = MAKE_TEST(PMD, PMD, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS, 1317 "4MB mremap - Source PMD-aligned, Destination PMD-aligned"); 1318 1319 /* Src addr PUD aligned */ 1320 test_cases[9] = MAKE_TEST(PUD, PTE, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS, 1321 "2GB mremap - Source PUD-aligned, Destination PTE-aligned"); 1322 test_cases[10] = MAKE_TEST(PUD, _1MB, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS, 1323 "2GB mremap - Source PUD-aligned, Destination 1MB-aligned"); 1324 test_cases[11] = MAKE_TEST(PUD, PMD, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS, 1325 "2GB mremap - Source PUD-aligned, Destination PMD-aligned"); 1326 test_cases[12] = MAKE_TEST(PUD, PUD, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS, 1327 "2GB mremap - Source PUD-aligned, Destination PUD-aligned"); 1328 1329 /* Src and Dest addr 1MB aligned. 5MB mremap. */ 1330 test_cases[13] = MAKE_TEST(_1MB, _1MB, _5MB, NON_OVERLAPPING, EXPECT_SUCCESS, 1331 "5MB mremap - Source 1MB-aligned, Destination 1MB-aligned"); 1332 1333 /* Src and Dest addr 1MB aligned. 5MB mremap. */ 1334 test_cases[14] = MAKE_TEST(_1MB, _1MB, _5MB, NON_OVERLAPPING, EXPECT_SUCCESS, 1335 "5MB mremap - Source 1MB-aligned, Dest 1MB-aligned with 40MB Preamble"); 1336 test_cases[14].config.dest_preamble_size = 10 * _4MB; 1337 1338 perf_test_cases[0] = MAKE_TEST(page_size, page_size, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS, 1339 "1GB mremap - Source PTE-aligned, Destination PTE-aligned"); 1340 /* 1341 * mremap 1GB region - Page table level aligned time 1342 * comparison. 1343 */ 1344 perf_test_cases[1] = MAKE_TEST(PMD, PMD, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS, 1345 "1GB mremap - Source PMD-aligned, Destination PMD-aligned"); 1346 perf_test_cases[2] = MAKE_TEST(PUD, PUD, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS, 1347 "1GB mremap - Source PUD-aligned, Destination PUD-aligned"); 1348 1349 run_perf_tests = (threshold_mb == VALIDATION_NO_THRESHOLD) || 1350 (threshold_mb * _1MB >= _1GB); 1351 1352 ksft_set_plan(ARRAY_SIZE(test_cases) + (run_perf_tests ? 1353 ARRAY_SIZE(perf_test_cases) : 0) + num_expand_tests + num_misc_tests); 1354 1355 for (i = 0; i < ARRAY_SIZE(test_cases); i++) 1356 run_mremap_test_case(test_cases[i], &failures, threshold_mb, 1357 rand_addr); 1358 1359 maps_fp = fopen("/proc/self/maps", "r"); 1360 1361 if (maps_fp == NULL) { 1362 munmap(rand_addr, rand_size); 1363 ksft_exit_fail_msg("Failed to read /proc/self/maps: %s\n", strerror(errno)); 1364 } 1365 1366 mremap_expand_merge(maps_fp, page_size); 1367 mremap_expand_merge_offset(maps_fp, page_size); 1368 1369 mremap_move_within_range(pattern_seed, rand_addr); 1370 mremap_move_1mb_from_start(pattern_seed, rand_addr); 1371 mremap_shrink_multiple_vmas(page_size, /* inplace= */true); 1372 mremap_shrink_multiple_vmas(page_size, /* inplace= */false); 1373 mremap_move_multiple_vmas(pattern_seed, page_size, /* dontunmap= */ false); 1374 mremap_move_multiple_vmas(pattern_seed, page_size, /* dontunmap= */ true); 1375 mremap_move_multiple_vmas_split(pattern_seed, page_size, /* dontunmap= */ false); 1376 mremap_move_multiple_vmas_split(pattern_seed, page_size, /* dontunmap= */ true); 1377 mremap_move_multi_invalid_vmas(maps_fp, page_size); 1378 1379 fclose(maps_fp); 1380 1381 if (run_perf_tests) { 1382 ksft_print_msg("\n%s\n", 1383 "mremap HAVE_MOVE_PMD/PUD optimization time comparison for 1GB region:"); 1384 for (i = 0; i < ARRAY_SIZE(perf_test_cases); i++) 1385 run_mremap_test_case(perf_test_cases[i], &failures, 1386 threshold_mb, 1387 rand_addr); 1388 } 1389 1390 munmap(rand_addr, rand_size); 1391 1392 if (failures > 0) 1393 ksft_exit_fail(); 1394 else 1395 ksft_exit_pass(); 1396 } 1397