1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright 2020 Google LLC 4 */ 5 #define _GNU_SOURCE 6 7 #include <errno.h> 8 #include <stdlib.h> 9 #include <stdio.h> 10 #include <string.h> 11 #include <sys/mman.h> 12 #include <time.h> 13 #include <stdbool.h> 14 15 #include "../kselftest.h" 16 17 #define EXPECT_SUCCESS 0 18 #define EXPECT_FAILURE 1 19 #define NON_OVERLAPPING 0 20 #define OVERLAPPING 1 21 #define NS_PER_SEC 1000000000ULL 22 #define VALIDATION_DEFAULT_THRESHOLD 4 /* 4MB */ 23 #define VALIDATION_NO_THRESHOLD 0 /* Verify the entire region */ 24 25 #ifndef MIN 26 #define MIN(X, Y) ((X) < (Y) ? (X) : (Y)) 27 #define MAX(X, Y) ((X) > (Y) ? (X) : (Y)) 28 #endif 29 #define SIZE_MB(m) ((size_t)m * (1024 * 1024)) 30 #define SIZE_KB(k) ((size_t)k * 1024) 31 32 struct config { 33 unsigned long long src_alignment; 34 unsigned long long dest_alignment; 35 unsigned long long region_size; 36 int overlapping; 37 unsigned int dest_preamble_size; 38 }; 39 40 struct test { 41 const char *name; 42 struct config config; 43 int expect_failure; 44 }; 45 46 enum { 47 _1KB = 1ULL << 10, /* 1KB -> not page aligned */ 48 _4KB = 4ULL << 10, 49 _8KB = 8ULL << 10, 50 _1MB = 1ULL << 20, 51 _2MB = 2ULL << 20, 52 _4MB = 4ULL << 20, 53 _5MB = 5ULL << 20, 54 _1GB = 1ULL << 30, 55 _2GB = 2ULL << 30, 56 PMD = _2MB, 57 PUD = _1GB, 58 }; 59 60 #define PTE page_size 61 62 #define MAKE_TEST(source_align, destination_align, size, \ 63 overlaps, should_fail, test_name) \ 64 (struct test){ \ 65 .name = test_name, \ 66 .config = { \ 67 .src_alignment = source_align, \ 68 .dest_alignment = destination_align, \ 69 .region_size = size, \ 70 .overlapping = overlaps, \ 71 }, \ 72 .expect_failure = should_fail \ 73 } 74 75 /* compute square root using binary search */ 76 static unsigned long get_sqrt(unsigned long val) 77 { 78 unsigned long low = 1; 79 80 /* assuming rand_size is less than 1TB */ 81 unsigned long high = (1UL << 20); 82 83 while (low <= high) { 84 unsigned long mid = low + (high - low) / 2; 85 unsigned long temp = mid * mid; 86 87 if (temp == val) 88 return mid; 89 if (temp < val) 90 low = mid + 1; 91 high = mid - 1; 92 } 93 return low; 94 } 95 96 /* 97 * Returns false if the requested remap region overlaps with an 98 * existing mapping (e.g text, stack) else returns true. 99 */ 100 static bool is_remap_region_valid(void *addr, unsigned long long size) 101 { 102 void *remap_addr = NULL; 103 bool ret = true; 104 105 /* Use MAP_FIXED_NOREPLACE flag to ensure region is not mapped */ 106 remap_addr = mmap(addr, size, PROT_READ | PROT_WRITE, 107 MAP_FIXED_NOREPLACE | MAP_ANONYMOUS | MAP_SHARED, 108 -1, 0); 109 110 if (remap_addr == MAP_FAILED) { 111 if (errno == EEXIST) 112 ret = false; 113 } else { 114 munmap(remap_addr, size); 115 } 116 117 return ret; 118 } 119 120 /* Returns mmap_min_addr sysctl tunable from procfs */ 121 static unsigned long long get_mmap_min_addr(void) 122 { 123 FILE *fp; 124 int n_matched; 125 static unsigned long long addr; 126 127 if (addr) 128 return addr; 129 130 fp = fopen("/proc/sys/vm/mmap_min_addr", "r"); 131 if (fp == NULL) { 132 ksft_print_msg("Failed to open /proc/sys/vm/mmap_min_addr: %s\n", 133 strerror(errno)); 134 exit(KSFT_SKIP); 135 } 136 137 n_matched = fscanf(fp, "%llu", &addr); 138 if (n_matched != 1) { 139 ksft_print_msg("Failed to read /proc/sys/vm/mmap_min_addr: %s\n", 140 strerror(errno)); 141 fclose(fp); 142 exit(KSFT_SKIP); 143 } 144 145 fclose(fp); 146 return addr; 147 } 148 149 /* 150 * Using /proc/self/maps, assert that the specified address range is contained 151 * within a single mapping. 152 */ 153 static bool is_range_mapped(FILE *maps_fp, unsigned long start, 154 unsigned long end) 155 { 156 char *line = NULL; 157 size_t len = 0; 158 bool success = false; 159 unsigned long first_val, second_val; 160 161 rewind(maps_fp); 162 163 while (getline(&line, &len, maps_fp) != -1) { 164 if (sscanf(line, "%lx-%lx", &first_val, &second_val) != 2) { 165 ksft_exit_fail_msg("cannot parse /proc/self/maps\n"); 166 break; 167 } 168 169 if (first_val <= start && second_val >= end) { 170 success = true; 171 break; 172 } 173 } 174 175 return success; 176 } 177 178 /* 179 * Returns the start address of the mapping on success, else returns 180 * NULL on failure. 181 */ 182 static void *get_source_mapping(struct config c) 183 { 184 unsigned long long addr = 0ULL; 185 void *src_addr = NULL; 186 unsigned long long mmap_min_addr; 187 188 mmap_min_addr = get_mmap_min_addr(); 189 /* 190 * For some tests, we need to not have any mappings below the 191 * source mapping. Add some headroom to mmap_min_addr for this. 192 */ 193 mmap_min_addr += 10 * _4MB; 194 195 retry: 196 addr += c.src_alignment; 197 if (addr < mmap_min_addr) 198 goto retry; 199 200 src_addr = mmap((void *) addr, c.region_size, PROT_READ | PROT_WRITE, 201 MAP_FIXED_NOREPLACE | MAP_ANONYMOUS | MAP_SHARED, 202 -1, 0); 203 if (src_addr == MAP_FAILED) { 204 if (errno == EPERM || errno == EEXIST) 205 goto retry; 206 goto error; 207 } 208 /* 209 * Check that the address is aligned to the specified alignment. 210 * Addresses which have alignments that are multiples of that 211 * specified are not considered valid. For instance, 1GB address is 212 * 2MB-aligned, however it will not be considered valid for a 213 * requested alignment of 2MB. This is done to reduce coincidental 214 * alignment in the tests. 215 */ 216 if (((unsigned long long) src_addr & (c.src_alignment - 1)) || 217 !((unsigned long long) src_addr & c.src_alignment)) { 218 munmap(src_addr, c.region_size); 219 goto retry; 220 } 221 222 if (!src_addr) 223 goto error; 224 225 return src_addr; 226 error: 227 ksft_print_msg("Failed to map source region: %s\n", 228 strerror(errno)); 229 return NULL; 230 } 231 232 /* 233 * This test validates that merge is called when expanding a mapping. 234 * Mapping containing three pages is created, middle page is unmapped 235 * and then the mapping containing the first page is expanded so that 236 * it fills the created hole. The two parts should merge creating 237 * single mapping with three pages. 238 */ 239 static void mremap_expand_merge(FILE *maps_fp, unsigned long page_size) 240 { 241 char *test_name = "mremap expand merge"; 242 bool success = false; 243 char *remap, *start; 244 245 start = mmap(NULL, 3 * page_size, PROT_READ | PROT_WRITE, 246 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); 247 248 if (start == MAP_FAILED) { 249 ksft_print_msg("mmap failed: %s\n", strerror(errno)); 250 goto out; 251 } 252 253 munmap(start + page_size, page_size); 254 remap = mremap(start, page_size, 2 * page_size, 0); 255 if (remap == MAP_FAILED) { 256 ksft_print_msg("mremap failed: %s\n", strerror(errno)); 257 munmap(start, page_size); 258 munmap(start + 2 * page_size, page_size); 259 goto out; 260 } 261 262 success = is_range_mapped(maps_fp, (unsigned long)start, 263 (unsigned long)(start + 3 * page_size)); 264 munmap(start, 3 * page_size); 265 266 out: 267 if (success) 268 ksft_test_result_pass("%s\n", test_name); 269 else 270 ksft_test_result_fail("%s\n", test_name); 271 } 272 273 /* 274 * Similar to mremap_expand_merge() except instead of removing the middle page, 275 * we remove the last then attempt to remap offset from the second page. This 276 * should result in the mapping being restored to its former state. 277 */ 278 static void mremap_expand_merge_offset(FILE *maps_fp, unsigned long page_size) 279 { 280 281 char *test_name = "mremap expand merge offset"; 282 bool success = false; 283 char *remap, *start; 284 285 start = mmap(NULL, 3 * page_size, PROT_READ | PROT_WRITE, 286 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); 287 288 if (start == MAP_FAILED) { 289 ksft_print_msg("mmap failed: %s\n", strerror(errno)); 290 goto out; 291 } 292 293 /* Unmap final page to ensure we have space to expand. */ 294 munmap(start + 2 * page_size, page_size); 295 remap = mremap(start + page_size, page_size, 2 * page_size, 0); 296 if (remap == MAP_FAILED) { 297 ksft_print_msg("mremap failed: %s\n", strerror(errno)); 298 munmap(start, 2 * page_size); 299 goto out; 300 } 301 302 success = is_range_mapped(maps_fp, (unsigned long)start, 303 (unsigned long)(start + 3 * page_size)); 304 munmap(start, 3 * page_size); 305 306 out: 307 if (success) 308 ksft_test_result_pass("%s\n", test_name); 309 else 310 ksft_test_result_fail("%s\n", test_name); 311 } 312 313 /* 314 * Verify that an mremap within a range does not cause corruption 315 * of unrelated part of range. 316 * 317 * Consider the following range which is 2MB aligned and is 318 * a part of a larger 20MB range which is not shown. Each 319 * character is 256KB below making the source and destination 320 * 2MB each. The lower case letters are moved (s to d) and the 321 * upper case letters are not moved. The below test verifies 322 * that the upper case S letters are not corrupted by the 323 * adjacent mremap. 324 * 325 * |DDDDddddSSSSssss| 326 */ 327 static void mremap_move_within_range(unsigned int pattern_seed, char *rand_addr) 328 { 329 char *test_name = "mremap mremap move within range"; 330 void *src, *dest; 331 unsigned int i, success = 1; 332 333 size_t size = SIZE_MB(20); 334 void *ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, 335 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); 336 if (ptr == MAP_FAILED) { 337 perror("mmap"); 338 success = 0; 339 goto out; 340 } 341 memset(ptr, 0, size); 342 343 src = ptr + SIZE_MB(6); 344 src = (void *)((unsigned long)src & ~(SIZE_MB(2) - 1)); 345 346 /* Set byte pattern for source block. */ 347 memcpy(src, rand_addr, SIZE_MB(2)); 348 349 dest = src - SIZE_MB(2); 350 351 void *new_ptr = mremap(src + SIZE_MB(1), SIZE_MB(1), SIZE_MB(1), 352 MREMAP_MAYMOVE | MREMAP_FIXED, dest + SIZE_MB(1)); 353 if (new_ptr == MAP_FAILED) { 354 perror("mremap"); 355 success = 0; 356 goto out; 357 } 358 359 /* Verify byte pattern after remapping */ 360 srand(pattern_seed); 361 for (i = 0; i < SIZE_MB(1); i++) { 362 char c = (char) rand(); 363 364 if (((char *)src)[i] != c) { 365 ksft_print_msg("Data at src at %d got corrupted due to unrelated mremap\n", 366 i); 367 ksft_print_msg("Expected: %#x\t Got: %#x\n", c & 0xff, 368 ((char *) src)[i] & 0xff); 369 success = 0; 370 } 371 } 372 373 out: 374 if (munmap(ptr, size) == -1) 375 perror("munmap"); 376 377 if (success) 378 ksft_test_result_pass("%s\n", test_name); 379 else 380 ksft_test_result_fail("%s\n", test_name); 381 } 382 383 static bool is_multiple_vma_range_ok(unsigned int pattern_seed, 384 char *ptr, unsigned long page_size) 385 { 386 int i; 387 388 srand(pattern_seed); 389 for (i = 0; i <= 10; i += 2) { 390 int j; 391 char *buf = &ptr[i * page_size]; 392 size_t size = i == 4 ? 2 * page_size : page_size; 393 394 for (j = 0; j < size; j++) { 395 char chr = rand(); 396 397 if (chr != buf[j]) { 398 ksft_print_msg("page %d offset %d corrupted, expected %d got %d\n", 399 i, j, chr, buf[j]); 400 return false; 401 } 402 } 403 } 404 405 return true; 406 } 407 408 static void mremap_move_multiple_vmas(unsigned int pattern_seed, 409 unsigned long page_size, 410 bool dont_unmap) 411 { 412 int mremap_flags = MREMAP_FIXED | MREMAP_MAYMOVE; 413 char *test_name = "mremap move multiple vmas"; 414 const size_t size = 11 * page_size; 415 bool success = true; 416 char *ptr, *tgt_ptr; 417 int i; 418 419 if (dont_unmap) 420 mremap_flags |= MREMAP_DONTUNMAP; 421 422 ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, 423 MAP_PRIVATE | MAP_ANON, -1, 0); 424 if (ptr == MAP_FAILED) { 425 perror("mmap"); 426 success = false; 427 goto out; 428 } 429 430 tgt_ptr = mmap(NULL, 2 * size, PROT_READ | PROT_WRITE, 431 MAP_PRIVATE | MAP_ANON, -1, 0); 432 if (tgt_ptr == MAP_FAILED) { 433 perror("mmap"); 434 success = false; 435 goto out; 436 } 437 if (munmap(tgt_ptr, 2 * size)) { 438 perror("munmap"); 439 success = false; 440 goto out_unmap; 441 } 442 443 /* 444 * Unmap so we end up with: 445 * 446 * 0 2 4 5 6 8 10 offset in buffer 447 * |*| |*| |*****| |*| |*| 448 * |*| |*| |*****| |*| |*| 449 * 0 1 2 3 4 5 6 pattern offset 450 */ 451 for (i = 1; i < 10; i += 2) { 452 if (i == 5) 453 continue; 454 455 if (munmap(&ptr[i * page_size], page_size)) { 456 perror("munmap"); 457 success = false; 458 goto out_unmap; 459 } 460 } 461 462 srand(pattern_seed); 463 464 /* Set up random patterns. */ 465 for (i = 0; i <= 10; i += 2) { 466 int j; 467 size_t size = i == 4 ? 2 * page_size : page_size; 468 char *buf = &ptr[i * page_size]; 469 470 for (j = 0; j < size; j++) 471 buf[j] = rand(); 472 } 473 474 /* First, just move the whole thing. */ 475 if (mremap(ptr, size, size, mremap_flags, tgt_ptr) == MAP_FAILED) { 476 perror("mremap"); 477 success = false; 478 goto out_unmap; 479 } 480 /* Check move was ok. */ 481 if (!is_multiple_vma_range_ok(pattern_seed, tgt_ptr, page_size)) { 482 success = false; 483 goto out_unmap; 484 } 485 486 /* Move next to itself. */ 487 if (mremap(tgt_ptr, size, size, mremap_flags, 488 &tgt_ptr[size]) == MAP_FAILED) { 489 perror("mremap"); 490 success = false; 491 goto out_unmap; 492 } 493 /* Check that the move is ok. */ 494 if (!is_multiple_vma_range_ok(pattern_seed, &tgt_ptr[size], page_size)) { 495 success = false; 496 goto out_unmap; 497 } 498 499 /* Map a range to overwrite. */ 500 if (mmap(tgt_ptr, size, PROT_NONE, 501 MAP_PRIVATE | MAP_ANON | MAP_FIXED, -1, 0) == MAP_FAILED) { 502 perror("mmap tgt"); 503 success = false; 504 goto out_unmap; 505 } 506 /* Move and overwrite. */ 507 if (mremap(&tgt_ptr[size], size, size, 508 mremap_flags, tgt_ptr) == MAP_FAILED) { 509 perror("mremap"); 510 success = false; 511 goto out_unmap; 512 } 513 /* Check that the move is ok. */ 514 if (!is_multiple_vma_range_ok(pattern_seed, tgt_ptr, page_size)) { 515 success = false; 516 goto out_unmap; 517 } 518 519 out_unmap: 520 if (munmap(tgt_ptr, 2 * size)) 521 perror("munmap tgt"); 522 if (munmap(ptr, size)) 523 perror("munmap src"); 524 525 out: 526 if (success) 527 ksft_test_result_pass("%s%s\n", test_name, 528 dont_unmap ? " [dontunnmap]" : ""); 529 else 530 ksft_test_result_fail("%s%s\n", test_name, 531 dont_unmap ? " [dontunnmap]" : ""); 532 } 533 534 static void mremap_shrink_multiple_vmas(unsigned long page_size, 535 bool inplace) 536 { 537 char *test_name = "mremap shrink multiple vmas"; 538 const size_t size = 10 * page_size; 539 bool success = true; 540 char *ptr, *tgt_ptr; 541 void *res; 542 int i; 543 544 ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, 545 MAP_PRIVATE | MAP_ANON, -1, 0); 546 if (ptr == MAP_FAILED) { 547 perror("mmap"); 548 success = false; 549 goto out; 550 } 551 552 tgt_ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, 553 MAP_PRIVATE | MAP_ANON, -1, 0); 554 if (tgt_ptr == MAP_FAILED) { 555 perror("mmap"); 556 success = false; 557 goto out; 558 } 559 if (munmap(tgt_ptr, size)) { 560 perror("munmap"); 561 success = false; 562 goto out_unmap; 563 } 564 565 /* 566 * Unmap so we end up with: 567 * 568 * 0 2 4 6 8 10 offset in buffer 569 * |*| |*| |*| |*| |*| |*| 570 * |*| |*| |*| |*| |*| |*| 571 */ 572 for (i = 1; i < 10; i += 2) { 573 if (munmap(&ptr[i * page_size], page_size)) { 574 perror("munmap"); 575 success = false; 576 goto out_unmap; 577 } 578 } 579 580 /* 581 * Shrink in-place across multiple VMAs and gaps so we end up with: 582 * 583 * 0 584 * |*| 585 * |*| 586 */ 587 if (inplace) 588 res = mremap(ptr, size, page_size, 0); 589 else 590 res = mremap(ptr, size, page_size, MREMAP_MAYMOVE | MREMAP_FIXED, 591 tgt_ptr); 592 593 if (res == MAP_FAILED) { 594 perror("mremap"); 595 success = false; 596 goto out_unmap; 597 } 598 599 out_unmap: 600 if (munmap(tgt_ptr, size)) 601 perror("munmap tgt"); 602 if (munmap(ptr, size)) 603 perror("munmap src"); 604 out: 605 if (success) 606 ksft_test_result_pass("%s%s\n", test_name, 607 inplace ? " [inplace]" : ""); 608 else 609 ksft_test_result_fail("%s%s\n", test_name, 610 inplace ? " [inplace]" : ""); 611 } 612 613 static void mremap_move_multiple_vmas_split(unsigned int pattern_seed, 614 unsigned long page_size, 615 bool dont_unmap) 616 { 617 char *test_name = "mremap move multiple vmas split"; 618 int mremap_flags = MREMAP_FIXED | MREMAP_MAYMOVE; 619 const size_t size = 10 * page_size; 620 bool success = true; 621 char *ptr, *tgt_ptr; 622 int i; 623 624 if (dont_unmap) 625 mremap_flags |= MREMAP_DONTUNMAP; 626 627 ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, 628 MAP_PRIVATE | MAP_ANON, -1, 0); 629 if (ptr == MAP_FAILED) { 630 perror("mmap"); 631 success = false; 632 goto out; 633 } 634 635 tgt_ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, 636 MAP_PRIVATE | MAP_ANON, -1, 0); 637 if (tgt_ptr == MAP_FAILED) { 638 perror("mmap"); 639 success = false; 640 goto out; 641 } 642 if (munmap(tgt_ptr, size)) { 643 perror("munmap"); 644 success = false; 645 goto out_unmap; 646 } 647 648 /* 649 * Unmap so we end up with: 650 * 651 * 0 1 2 3 4 5 6 7 8 9 10 offset in buffer 652 * |**********| |*******| 653 * |**********| |*******| 654 * 0 1 2 3 4 5 6 7 8 9 pattern offset 655 */ 656 if (munmap(&ptr[5 * page_size], page_size)) { 657 perror("munmap"); 658 success = false; 659 goto out_unmap; 660 } 661 662 /* Set up random patterns. */ 663 srand(pattern_seed); 664 for (i = 0; i < 10; i++) { 665 int j; 666 char *buf = &ptr[i * page_size]; 667 668 if (i == 5) 669 continue; 670 671 for (j = 0; j < page_size; j++) 672 buf[j] = rand(); 673 } 674 675 /* 676 * Move the below: 677 * 678 * <-------------> 679 * 0 1 2 3 4 5 6 7 8 9 10 offset in buffer 680 * |**********| |*******| 681 * |**********| |*******| 682 * 0 1 2 3 4 5 6 7 8 9 pattern offset 683 * 684 * Into: 685 * 686 * 0 1 2 3 4 5 6 7 offset in buffer 687 * |*****| |*****| 688 * |*****| |*****| 689 * 2 3 4 5 6 7 pattern offset 690 */ 691 if (mremap(&ptr[2 * page_size], size - 3 * page_size, size - 3 * page_size, 692 mremap_flags, tgt_ptr) == MAP_FAILED) { 693 perror("mremap"); 694 success = false; 695 goto out_unmap; 696 } 697 698 /* Offset into random pattern. */ 699 srand(pattern_seed); 700 for (i = 0; i < 2 * page_size; i++) 701 rand(); 702 703 /* Check pattern. */ 704 for (i = 0; i < 7; i++) { 705 int j; 706 char *buf = &tgt_ptr[i * page_size]; 707 708 if (i == 3) 709 continue; 710 711 for (j = 0; j < page_size; j++) { 712 char chr = rand(); 713 714 if (chr != buf[j]) { 715 ksft_print_msg("page %d offset %d corrupted, expected %d got %d\n", 716 i, j, chr, buf[j]); 717 goto out_unmap; 718 } 719 } 720 } 721 722 out_unmap: 723 if (munmap(tgt_ptr, size)) 724 perror("munmap tgt"); 725 if (munmap(ptr, size)) 726 perror("munmap src"); 727 out: 728 if (success) 729 ksft_test_result_pass("%s%s\n", test_name, 730 dont_unmap ? " [dontunnmap]" : ""); 731 else 732 ksft_test_result_fail("%s%s\n", test_name, 733 dont_unmap ? " [dontunnmap]" : ""); 734 } 735 736 /* Returns the time taken for the remap on success else returns -1. */ 737 static long long remap_region(struct config c, unsigned int threshold_mb, 738 char *rand_addr) 739 { 740 void *addr, *src_addr, *dest_addr, *dest_preamble_addr = NULL; 741 unsigned long long t, d; 742 struct timespec t_start = {0, 0}, t_end = {0, 0}; 743 long long start_ns, end_ns, align_mask, ret, offset; 744 unsigned long long threshold; 745 unsigned long num_chunks; 746 747 if (threshold_mb == VALIDATION_NO_THRESHOLD) 748 threshold = c.region_size; 749 else 750 threshold = MIN(threshold_mb * _1MB, c.region_size); 751 752 src_addr = get_source_mapping(c); 753 if (!src_addr) { 754 ret = -1; 755 goto out; 756 } 757 758 /* Set byte pattern for source block. */ 759 memcpy(src_addr, rand_addr, threshold); 760 761 /* Mask to zero out lower bits of address for alignment */ 762 align_mask = ~(c.dest_alignment - 1); 763 /* Offset of destination address from the end of the source region */ 764 offset = (c.overlapping) ? -c.dest_alignment : c.dest_alignment; 765 addr = (void *) (((unsigned long long) src_addr + c.region_size 766 + offset) & align_mask); 767 768 /* Remap after the destination block preamble. */ 769 addr += c.dest_preamble_size; 770 771 /* See comment in get_source_mapping() */ 772 if (!((unsigned long long) addr & c.dest_alignment)) 773 addr = (void *) ((unsigned long long) addr | c.dest_alignment); 774 775 /* Don't destroy existing mappings unless expected to overlap */ 776 while (!is_remap_region_valid(addr, c.region_size) && !c.overlapping) { 777 /* Check for unsigned overflow */ 778 if (addr + c.dest_alignment < addr) { 779 ksft_print_msg("Couldn't find a valid region to remap to\n"); 780 ret = -1; 781 goto clean_up_src; 782 } 783 addr += c.dest_alignment; 784 } 785 786 if (c.dest_preamble_size) { 787 dest_preamble_addr = mmap((void *) addr - c.dest_preamble_size, c.dest_preamble_size, 788 PROT_READ | PROT_WRITE, 789 MAP_FIXED_NOREPLACE | MAP_ANONYMOUS | MAP_SHARED, 790 -1, 0); 791 if (dest_preamble_addr == MAP_FAILED) { 792 ksft_print_msg("Failed to map dest preamble region: %s\n", 793 strerror(errno)); 794 ret = -1; 795 goto clean_up_src; 796 } 797 798 /* Set byte pattern for the dest preamble block. */ 799 memcpy(dest_preamble_addr, rand_addr, c.dest_preamble_size); 800 } 801 802 clock_gettime(CLOCK_MONOTONIC, &t_start); 803 dest_addr = mremap(src_addr, c.region_size, c.region_size, 804 MREMAP_MAYMOVE|MREMAP_FIXED, (char *) addr); 805 clock_gettime(CLOCK_MONOTONIC, &t_end); 806 807 if (dest_addr == MAP_FAILED) { 808 ksft_print_msg("mremap failed: %s\n", strerror(errno)); 809 ret = -1; 810 goto clean_up_dest_preamble; 811 } 812 813 /* 814 * Verify byte pattern after remapping. Employ an algorithm with a 815 * square root time complexity in threshold: divide the range into 816 * chunks, if memcmp() returns non-zero, only then perform an 817 * iteration in that chunk to find the mismatch index. 818 */ 819 num_chunks = get_sqrt(threshold); 820 for (unsigned long i = 0; i < num_chunks; ++i) { 821 size_t chunk_size = threshold / num_chunks; 822 unsigned long shift = i * chunk_size; 823 824 if (!memcmp(dest_addr + shift, rand_addr + shift, chunk_size)) 825 continue; 826 827 /* brute force iteration only over mismatch segment */ 828 for (t = shift; t < shift + chunk_size; ++t) { 829 if (((char *) dest_addr)[t] != rand_addr[t]) { 830 ksft_print_msg("Data after remap doesn't match at offset %llu\n", 831 t); 832 ksft_print_msg("Expected: %#x\t Got: %#x\n", rand_addr[t] & 0xff, 833 ((char *) dest_addr)[t] & 0xff); 834 ret = -1; 835 goto clean_up_dest; 836 } 837 } 838 } 839 840 /* 841 * if threshold is not divisible by num_chunks, then check the 842 * last chunk 843 */ 844 for (t = num_chunks * (threshold / num_chunks); t < threshold; ++t) { 845 if (((char *) dest_addr)[t] != rand_addr[t]) { 846 ksft_print_msg("Data after remap doesn't match at offset %llu\n", 847 t); 848 ksft_print_msg("Expected: %#x\t Got: %#x\n", rand_addr[t] & 0xff, 849 ((char *) dest_addr)[t] & 0xff); 850 ret = -1; 851 goto clean_up_dest; 852 } 853 } 854 855 /* Verify the dest preamble byte pattern after remapping */ 856 if (!c.dest_preamble_size) 857 goto no_preamble; 858 859 num_chunks = get_sqrt(c.dest_preamble_size); 860 861 for (unsigned long i = 0; i < num_chunks; ++i) { 862 size_t chunk_size = c.dest_preamble_size / num_chunks; 863 unsigned long shift = i * chunk_size; 864 865 if (!memcmp(dest_preamble_addr + shift, rand_addr + shift, 866 chunk_size)) 867 continue; 868 869 /* brute force iteration only over mismatched segment */ 870 for (d = shift; d < shift + chunk_size; ++d) { 871 if (((char *) dest_preamble_addr)[d] != rand_addr[d]) { 872 ksft_print_msg("Preamble data after remap doesn't match at offset %llu\n", 873 d); 874 ksft_print_msg("Expected: %#x\t Got: %#x\n", rand_addr[d] & 0xff, 875 ((char *) dest_preamble_addr)[d] & 0xff); 876 ret = -1; 877 goto clean_up_dest; 878 } 879 } 880 } 881 882 for (d = num_chunks * (c.dest_preamble_size / num_chunks); d < c.dest_preamble_size; ++d) { 883 if (((char *) dest_preamble_addr)[d] != rand_addr[d]) { 884 ksft_print_msg("Preamble data after remap doesn't match at offset %llu\n", 885 d); 886 ksft_print_msg("Expected: %#x\t Got: %#x\n", rand_addr[d] & 0xff, 887 ((char *) dest_preamble_addr)[d] & 0xff); 888 ret = -1; 889 goto clean_up_dest; 890 } 891 } 892 893 no_preamble: 894 start_ns = t_start.tv_sec * NS_PER_SEC + t_start.tv_nsec; 895 end_ns = t_end.tv_sec * NS_PER_SEC + t_end.tv_nsec; 896 ret = end_ns - start_ns; 897 898 /* 899 * Since the destination address is specified using MREMAP_FIXED, subsequent 900 * mremap will unmap any previous mapping at the address range specified by 901 * dest_addr and region_size. This significantly affects the remap time of 902 * subsequent tests. So we clean up mappings after each test. 903 */ 904 clean_up_dest: 905 munmap(dest_addr, c.region_size); 906 clean_up_dest_preamble: 907 if (c.dest_preamble_size && dest_preamble_addr) 908 munmap(dest_preamble_addr, c.dest_preamble_size); 909 clean_up_src: 910 munmap(src_addr, c.region_size); 911 out: 912 return ret; 913 } 914 915 /* 916 * Verify that an mremap aligning down does not destroy 917 * the beginning of the mapping just because the aligned 918 * down address landed on a mapping that maybe does not exist. 919 */ 920 static void mremap_move_1mb_from_start(unsigned int pattern_seed, 921 char *rand_addr) 922 { 923 char *test_name = "mremap move 1mb from start at 1MB+256KB aligned src"; 924 void *src = NULL, *dest = NULL; 925 unsigned int i, success = 1; 926 927 /* Config to reuse get_source_mapping() to do an aligned mmap. */ 928 struct config c = { 929 .src_alignment = SIZE_MB(1) + SIZE_KB(256), 930 .region_size = SIZE_MB(6) 931 }; 932 933 src = get_source_mapping(c); 934 if (!src) { 935 success = 0; 936 goto out; 937 } 938 939 c.src_alignment = SIZE_MB(1) + SIZE_KB(256); 940 dest = get_source_mapping(c); 941 if (!dest) { 942 success = 0; 943 goto out; 944 } 945 946 /* Set byte pattern for source block. */ 947 memcpy(src, rand_addr, SIZE_MB(2)); 948 949 /* 950 * Unmap the beginning of dest so that the aligned address 951 * falls on no mapping. 952 */ 953 munmap(dest, SIZE_MB(1)); 954 955 void *new_ptr = mremap(src + SIZE_MB(1), SIZE_MB(1), SIZE_MB(1), 956 MREMAP_MAYMOVE | MREMAP_FIXED, dest + SIZE_MB(1)); 957 if (new_ptr == MAP_FAILED) { 958 perror("mremap"); 959 success = 0; 960 goto out; 961 } 962 963 /* Verify byte pattern after remapping */ 964 srand(pattern_seed); 965 for (i = 0; i < SIZE_MB(1); i++) { 966 char c = (char) rand(); 967 968 if (((char *)src)[i] != c) { 969 ksft_print_msg("Data at src at %d got corrupted due to unrelated mremap\n", 970 i); 971 ksft_print_msg("Expected: %#x\t Got: %#x\n", c & 0xff, 972 ((char *) src)[i] & 0xff); 973 success = 0; 974 } 975 } 976 977 out: 978 if (src && munmap(src, c.region_size) == -1) 979 perror("munmap src"); 980 981 if (dest && munmap(dest, c.region_size) == -1) 982 perror("munmap dest"); 983 984 if (success) 985 ksft_test_result_pass("%s\n", test_name); 986 else 987 ksft_test_result_fail("%s\n", test_name); 988 } 989 990 static void run_mremap_test_case(struct test test_case, int *failures, 991 unsigned int threshold_mb, 992 char *rand_addr) 993 { 994 long long remap_time = remap_region(test_case.config, threshold_mb, 995 rand_addr); 996 997 if (remap_time < 0) { 998 if (test_case.expect_failure) 999 ksft_test_result_xfail("%s\n\tExpected mremap failure\n", 1000 test_case.name); 1001 else { 1002 ksft_test_result_fail("%s\n", test_case.name); 1003 *failures += 1; 1004 } 1005 } else { 1006 /* 1007 * Comparing mremap time is only applicable if entire region 1008 * was faulted in. 1009 */ 1010 if (threshold_mb == VALIDATION_NO_THRESHOLD || 1011 test_case.config.region_size <= threshold_mb * _1MB) 1012 ksft_test_result_pass("%s\n\tmremap time: %12lldns\n", 1013 test_case.name, remap_time); 1014 else 1015 ksft_test_result_pass("%s\n", test_case.name); 1016 } 1017 } 1018 1019 static void usage(const char *cmd) 1020 { 1021 fprintf(stderr, 1022 "Usage: %s [[-t <threshold_mb>] [-p <pattern_seed>]]\n" 1023 "-t\t only validate threshold_mb of the remapped region\n" 1024 " \t if 0 is supplied no threshold is used; all tests\n" 1025 " \t are run and remapped regions validated fully.\n" 1026 " \t The default threshold used is 4MB.\n" 1027 "-p\t provide a seed to generate the random pattern for\n" 1028 " \t validating the remapped region.\n", cmd); 1029 } 1030 1031 static int parse_args(int argc, char **argv, unsigned int *threshold_mb, 1032 unsigned int *pattern_seed) 1033 { 1034 const char *optstr = "t:p:"; 1035 int opt; 1036 1037 while ((opt = getopt(argc, argv, optstr)) != -1) { 1038 switch (opt) { 1039 case 't': 1040 *threshold_mb = atoi(optarg); 1041 break; 1042 case 'p': 1043 *pattern_seed = atoi(optarg); 1044 break; 1045 default: 1046 usage(argv[0]); 1047 return -1; 1048 } 1049 } 1050 1051 if (optind < argc) { 1052 usage(argv[0]); 1053 return -1; 1054 } 1055 1056 return 0; 1057 } 1058 1059 #define MAX_TEST 15 1060 #define MAX_PERF_TEST 3 1061 int main(int argc, char **argv) 1062 { 1063 int failures = 0; 1064 unsigned int i; 1065 int run_perf_tests; 1066 unsigned int threshold_mb = VALIDATION_DEFAULT_THRESHOLD; 1067 1068 /* hard-coded test configs */ 1069 size_t max_test_variable_region_size = _2GB; 1070 size_t max_test_constant_region_size = _2MB; 1071 size_t dest_preamble_size = 10 * _4MB; 1072 1073 unsigned int pattern_seed; 1074 char *rand_addr; 1075 size_t rand_size; 1076 int num_expand_tests = 2; 1077 int num_misc_tests = 8; 1078 struct test test_cases[MAX_TEST] = {}; 1079 struct test perf_test_cases[MAX_PERF_TEST]; 1080 int page_size; 1081 time_t t; 1082 FILE *maps_fp; 1083 1084 pattern_seed = (unsigned int) time(&t); 1085 1086 if (parse_args(argc, argv, &threshold_mb, &pattern_seed) < 0) 1087 exit(EXIT_FAILURE); 1088 1089 ksft_print_msg("Test configs:\n\tthreshold_mb=%u\n\tpattern_seed=%u\n\n", 1090 threshold_mb, pattern_seed); 1091 1092 /* 1093 * set preallocated random array according to test configs; see the 1094 * functions for the logic of setting the size 1095 */ 1096 if (!threshold_mb) 1097 rand_size = MAX(max_test_variable_region_size, 1098 max_test_constant_region_size); 1099 else 1100 rand_size = MAX(MIN(threshold_mb * _1MB, 1101 max_test_variable_region_size), 1102 max_test_constant_region_size); 1103 rand_size = MAX(dest_preamble_size, rand_size); 1104 1105 rand_addr = (char *)mmap(NULL, rand_size, PROT_READ | PROT_WRITE, 1106 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); 1107 if (rand_addr == MAP_FAILED) { 1108 perror("mmap"); 1109 ksft_exit_fail_msg("cannot mmap rand_addr\n"); 1110 } 1111 1112 /* fill stream of random bytes */ 1113 srand(pattern_seed); 1114 for (unsigned long i = 0; i < rand_size; ++i) 1115 rand_addr[i] = (char) rand(); 1116 1117 page_size = sysconf(_SC_PAGESIZE); 1118 1119 /* Expected mremap failures */ 1120 test_cases[0] = MAKE_TEST(page_size, page_size, page_size, 1121 OVERLAPPING, EXPECT_FAILURE, 1122 "mremap - Source and Destination Regions Overlapping"); 1123 1124 test_cases[1] = MAKE_TEST(page_size, page_size/4, page_size, 1125 NON_OVERLAPPING, EXPECT_FAILURE, 1126 "mremap - Destination Address Misaligned (1KB-aligned)"); 1127 test_cases[2] = MAKE_TEST(page_size/4, page_size, page_size, 1128 NON_OVERLAPPING, EXPECT_FAILURE, 1129 "mremap - Source Address Misaligned (1KB-aligned)"); 1130 1131 /* Src addr PTE aligned */ 1132 test_cases[3] = MAKE_TEST(PTE, PTE, PTE * 2, 1133 NON_OVERLAPPING, EXPECT_SUCCESS, 1134 "8KB mremap - Source PTE-aligned, Destination PTE-aligned"); 1135 1136 /* Src addr 1MB aligned */ 1137 test_cases[4] = MAKE_TEST(_1MB, PTE, _2MB, NON_OVERLAPPING, EXPECT_SUCCESS, 1138 "2MB mremap - Source 1MB-aligned, Destination PTE-aligned"); 1139 test_cases[5] = MAKE_TEST(_1MB, _1MB, _2MB, NON_OVERLAPPING, EXPECT_SUCCESS, 1140 "2MB mremap - Source 1MB-aligned, Destination 1MB-aligned"); 1141 1142 /* Src addr PMD aligned */ 1143 test_cases[6] = MAKE_TEST(PMD, PTE, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS, 1144 "4MB mremap - Source PMD-aligned, Destination PTE-aligned"); 1145 test_cases[7] = MAKE_TEST(PMD, _1MB, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS, 1146 "4MB mremap - Source PMD-aligned, Destination 1MB-aligned"); 1147 test_cases[8] = MAKE_TEST(PMD, PMD, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS, 1148 "4MB mremap - Source PMD-aligned, Destination PMD-aligned"); 1149 1150 /* Src addr PUD aligned */ 1151 test_cases[9] = MAKE_TEST(PUD, PTE, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS, 1152 "2GB mremap - Source PUD-aligned, Destination PTE-aligned"); 1153 test_cases[10] = MAKE_TEST(PUD, _1MB, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS, 1154 "2GB mremap - Source PUD-aligned, Destination 1MB-aligned"); 1155 test_cases[11] = MAKE_TEST(PUD, PMD, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS, 1156 "2GB mremap - Source PUD-aligned, Destination PMD-aligned"); 1157 test_cases[12] = MAKE_TEST(PUD, PUD, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS, 1158 "2GB mremap - Source PUD-aligned, Destination PUD-aligned"); 1159 1160 /* Src and Dest addr 1MB aligned. 5MB mremap. */ 1161 test_cases[13] = MAKE_TEST(_1MB, _1MB, _5MB, NON_OVERLAPPING, EXPECT_SUCCESS, 1162 "5MB mremap - Source 1MB-aligned, Destination 1MB-aligned"); 1163 1164 /* Src and Dest addr 1MB aligned. 5MB mremap. */ 1165 test_cases[14] = MAKE_TEST(_1MB, _1MB, _5MB, NON_OVERLAPPING, EXPECT_SUCCESS, 1166 "5MB mremap - Source 1MB-aligned, Dest 1MB-aligned with 40MB Preamble"); 1167 test_cases[14].config.dest_preamble_size = 10 * _4MB; 1168 1169 perf_test_cases[0] = MAKE_TEST(page_size, page_size, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS, 1170 "1GB mremap - Source PTE-aligned, Destination PTE-aligned"); 1171 /* 1172 * mremap 1GB region - Page table level aligned time 1173 * comparison. 1174 */ 1175 perf_test_cases[1] = MAKE_TEST(PMD, PMD, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS, 1176 "1GB mremap - Source PMD-aligned, Destination PMD-aligned"); 1177 perf_test_cases[2] = MAKE_TEST(PUD, PUD, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS, 1178 "1GB mremap - Source PUD-aligned, Destination PUD-aligned"); 1179 1180 run_perf_tests = (threshold_mb == VALIDATION_NO_THRESHOLD) || 1181 (threshold_mb * _1MB >= _1GB); 1182 1183 ksft_set_plan(ARRAY_SIZE(test_cases) + (run_perf_tests ? 1184 ARRAY_SIZE(perf_test_cases) : 0) + num_expand_tests + num_misc_tests); 1185 1186 for (i = 0; i < ARRAY_SIZE(test_cases); i++) 1187 run_mremap_test_case(test_cases[i], &failures, threshold_mb, 1188 rand_addr); 1189 1190 maps_fp = fopen("/proc/self/maps", "r"); 1191 1192 if (maps_fp == NULL) { 1193 munmap(rand_addr, rand_size); 1194 ksft_exit_fail_msg("Failed to read /proc/self/maps: %s\n", strerror(errno)); 1195 } 1196 1197 mremap_expand_merge(maps_fp, page_size); 1198 mremap_expand_merge_offset(maps_fp, page_size); 1199 1200 fclose(maps_fp); 1201 1202 mremap_move_within_range(pattern_seed, rand_addr); 1203 mremap_move_1mb_from_start(pattern_seed, rand_addr); 1204 mremap_shrink_multiple_vmas(page_size, /* inplace= */true); 1205 mremap_shrink_multiple_vmas(page_size, /* inplace= */false); 1206 mremap_move_multiple_vmas(pattern_seed, page_size, /* dontunmap= */ false); 1207 mremap_move_multiple_vmas(pattern_seed, page_size, /* dontunmap= */ true); 1208 mremap_move_multiple_vmas_split(pattern_seed, page_size, /* dontunmap= */ false); 1209 mremap_move_multiple_vmas_split(pattern_seed, page_size, /* dontunmap= */ true); 1210 1211 if (run_perf_tests) { 1212 ksft_print_msg("\n%s\n", 1213 "mremap HAVE_MOVE_PMD/PUD optimization time comparison for 1GB region:"); 1214 for (i = 0; i < ARRAY_SIZE(perf_test_cases); i++) 1215 run_mremap_test_case(perf_test_cases[i], &failures, 1216 threshold_mb, 1217 rand_addr); 1218 } 1219 1220 munmap(rand_addr, rand_size); 1221 1222 if (failures > 0) 1223 ksft_exit_fail(); 1224 else 1225 ksft_exit_pass(); 1226 } 1227