1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright 2020 Google LLC 4 */ 5 #define _GNU_SOURCE 6 7 #include <errno.h> 8 #include <stdlib.h> 9 #include <stdio.h> 10 #include <string.h> 11 #include <sys/mman.h> 12 #include <time.h> 13 #include <stdbool.h> 14 15 #include "../kselftest.h" 16 17 #define EXPECT_SUCCESS 0 18 #define EXPECT_FAILURE 1 19 #define NON_OVERLAPPING 0 20 #define OVERLAPPING 1 21 #define NS_PER_SEC 1000000000ULL 22 #define VALIDATION_DEFAULT_THRESHOLD 4 /* 4MB */ 23 #define VALIDATION_NO_THRESHOLD 0 /* Verify the entire region */ 24 25 #define MIN(X, Y) ((X) < (Y) ? (X) : (Y)) 26 #define SIZE_MB(m) ((size_t)m * (1024 * 1024)) 27 #define SIZE_KB(k) ((size_t)k * 1024) 28 29 struct config { 30 unsigned long long src_alignment; 31 unsigned long long dest_alignment; 32 unsigned long long region_size; 33 int overlapping; 34 int dest_preamble_size; 35 }; 36 37 struct test { 38 const char *name; 39 struct config config; 40 int expect_failure; 41 }; 42 43 enum { 44 _1KB = 1ULL << 10, /* 1KB -> not page aligned */ 45 _4KB = 4ULL << 10, 46 _8KB = 8ULL << 10, 47 _1MB = 1ULL << 20, 48 _2MB = 2ULL << 20, 49 _4MB = 4ULL << 20, 50 _5MB = 5ULL << 20, 51 _1GB = 1ULL << 30, 52 _2GB = 2ULL << 30, 53 PMD = _2MB, 54 PUD = _1GB, 55 }; 56 57 #define PTE page_size 58 59 #define MAKE_TEST(source_align, destination_align, size, \ 60 overlaps, should_fail, test_name) \ 61 (struct test){ \ 62 .name = test_name, \ 63 .config = { \ 64 .src_alignment = source_align, \ 65 .dest_alignment = destination_align, \ 66 .region_size = size, \ 67 .overlapping = overlaps, \ 68 }, \ 69 .expect_failure = should_fail \ 70 } 71 72 /* 73 * Returns false if the requested remap region overlaps with an 74 * existing mapping (e.g text, stack) else returns true. 75 */ 76 static bool is_remap_region_valid(void *addr, unsigned long long size) 77 { 78 void *remap_addr = NULL; 79 bool ret = true; 80 81 /* Use MAP_FIXED_NOREPLACE flag to ensure region is not mapped */ 82 remap_addr = mmap(addr, size, PROT_READ | PROT_WRITE, 83 MAP_FIXED_NOREPLACE | MAP_ANONYMOUS | MAP_SHARED, 84 -1, 0); 85 86 if (remap_addr == MAP_FAILED) { 87 if (errno == EEXIST) 88 ret = false; 89 } else { 90 munmap(remap_addr, size); 91 } 92 93 return ret; 94 } 95 96 /* Returns mmap_min_addr sysctl tunable from procfs */ 97 static unsigned long long get_mmap_min_addr(void) 98 { 99 FILE *fp; 100 int n_matched; 101 static unsigned long long addr; 102 103 if (addr) 104 return addr; 105 106 fp = fopen("/proc/sys/vm/mmap_min_addr", "r"); 107 if (fp == NULL) { 108 ksft_print_msg("Failed to open /proc/sys/vm/mmap_min_addr: %s\n", 109 strerror(errno)); 110 exit(KSFT_SKIP); 111 } 112 113 n_matched = fscanf(fp, "%llu", &addr); 114 if (n_matched != 1) { 115 ksft_print_msg("Failed to read /proc/sys/vm/mmap_min_addr: %s\n", 116 strerror(errno)); 117 fclose(fp); 118 exit(KSFT_SKIP); 119 } 120 121 fclose(fp); 122 return addr; 123 } 124 125 /* 126 * Using /proc/self/maps, assert that the specified address range is contained 127 * within a single mapping. 128 */ 129 static bool is_range_mapped(FILE *maps_fp, void *start, void *end) 130 { 131 char *line = NULL; 132 size_t len = 0; 133 bool success = false; 134 135 rewind(maps_fp); 136 137 while (getline(&line, &len, maps_fp) != -1) { 138 char *first = strtok(line, "- "); 139 void *first_val = (void *)strtol(first, NULL, 16); 140 char *second = strtok(NULL, "- "); 141 void *second_val = (void *) strtol(second, NULL, 16); 142 143 if (first_val <= start && second_val >= end) { 144 success = true; 145 break; 146 } 147 } 148 149 return success; 150 } 151 152 /* 153 * Returns the start address of the mapping on success, else returns 154 * NULL on failure. 155 */ 156 static void *get_source_mapping(struct config c) 157 { 158 unsigned long long addr = 0ULL; 159 void *src_addr = NULL; 160 unsigned long long mmap_min_addr; 161 162 mmap_min_addr = get_mmap_min_addr(); 163 /* 164 * For some tests, we need to not have any mappings below the 165 * source mapping. Add some headroom to mmap_min_addr for this. 166 */ 167 mmap_min_addr += 10 * _4MB; 168 169 retry: 170 addr += c.src_alignment; 171 if (addr < mmap_min_addr) 172 goto retry; 173 174 src_addr = mmap((void *) addr, c.region_size, PROT_READ | PROT_WRITE, 175 MAP_FIXED_NOREPLACE | MAP_ANONYMOUS | MAP_SHARED, 176 -1, 0); 177 if (src_addr == MAP_FAILED) { 178 if (errno == EPERM || errno == EEXIST) 179 goto retry; 180 goto error; 181 } 182 /* 183 * Check that the address is aligned to the specified alignment. 184 * Addresses which have alignments that are multiples of that 185 * specified are not considered valid. For instance, 1GB address is 186 * 2MB-aligned, however it will not be considered valid for a 187 * requested alignment of 2MB. This is done to reduce coincidental 188 * alignment in the tests. 189 */ 190 if (((unsigned long long) src_addr & (c.src_alignment - 1)) || 191 !((unsigned long long) src_addr & c.src_alignment)) { 192 munmap(src_addr, c.region_size); 193 goto retry; 194 } 195 196 if (!src_addr) 197 goto error; 198 199 return src_addr; 200 error: 201 ksft_print_msg("Failed to map source region: %s\n", 202 strerror(errno)); 203 return NULL; 204 } 205 206 /* 207 * This test validates that merge is called when expanding a mapping. 208 * Mapping containing three pages is created, middle page is unmapped 209 * and then the mapping containing the first page is expanded so that 210 * it fills the created hole. The two parts should merge creating 211 * single mapping with three pages. 212 */ 213 static void mremap_expand_merge(FILE *maps_fp, unsigned long page_size) 214 { 215 char *test_name = "mremap expand merge"; 216 bool success = false; 217 char *remap, *start; 218 219 start = mmap(NULL, 3 * page_size, PROT_READ | PROT_WRITE, 220 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); 221 222 if (start == MAP_FAILED) { 223 ksft_print_msg("mmap failed: %s\n", strerror(errno)); 224 goto out; 225 } 226 227 munmap(start + page_size, page_size); 228 remap = mremap(start, page_size, 2 * page_size, 0); 229 if (remap == MAP_FAILED) { 230 ksft_print_msg("mremap failed: %s\n", strerror(errno)); 231 munmap(start, page_size); 232 munmap(start + 2 * page_size, page_size); 233 goto out; 234 } 235 236 success = is_range_mapped(maps_fp, start, start + 3 * page_size); 237 munmap(start, 3 * page_size); 238 239 out: 240 if (success) 241 ksft_test_result_pass("%s\n", test_name); 242 else 243 ksft_test_result_fail("%s\n", test_name); 244 } 245 246 /* 247 * Similar to mremap_expand_merge() except instead of removing the middle page, 248 * we remove the last then attempt to remap offset from the second page. This 249 * should result in the mapping being restored to its former state. 250 */ 251 static void mremap_expand_merge_offset(FILE *maps_fp, unsigned long page_size) 252 { 253 254 char *test_name = "mremap expand merge offset"; 255 bool success = false; 256 char *remap, *start; 257 258 start = mmap(NULL, 3 * page_size, PROT_READ | PROT_WRITE, 259 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); 260 261 if (start == MAP_FAILED) { 262 ksft_print_msg("mmap failed: %s\n", strerror(errno)); 263 goto out; 264 } 265 266 /* Unmap final page to ensure we have space to expand. */ 267 munmap(start + 2 * page_size, page_size); 268 remap = mremap(start + page_size, page_size, 2 * page_size, 0); 269 if (remap == MAP_FAILED) { 270 ksft_print_msg("mremap failed: %s\n", strerror(errno)); 271 munmap(start, 2 * page_size); 272 goto out; 273 } 274 275 success = is_range_mapped(maps_fp, start, start + 3 * page_size); 276 munmap(start, 3 * page_size); 277 278 out: 279 if (success) 280 ksft_test_result_pass("%s\n", test_name); 281 else 282 ksft_test_result_fail("%s\n", test_name); 283 } 284 285 /* 286 * Verify that an mremap within a range does not cause corruption 287 * of unrelated part of range. 288 * 289 * Consider the following range which is 2MB aligned and is 290 * a part of a larger 20MB range which is not shown. Each 291 * character is 256KB below making the source and destination 292 * 2MB each. The lower case letters are moved (s to d) and the 293 * upper case letters are not moved. The below test verifies 294 * that the upper case S letters are not corrupted by the 295 * adjacent mremap. 296 * 297 * |DDDDddddSSSSssss| 298 */ 299 static void mremap_move_within_range(char pattern_seed) 300 { 301 char *test_name = "mremap mremap move within range"; 302 void *src, *dest; 303 int i, success = 1; 304 305 size_t size = SIZE_MB(20); 306 void *ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, 307 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); 308 if (ptr == MAP_FAILED) { 309 perror("mmap"); 310 success = 0; 311 goto out; 312 } 313 memset(ptr, 0, size); 314 315 src = ptr + SIZE_MB(6); 316 src = (void *)((unsigned long)src & ~(SIZE_MB(2) - 1)); 317 318 /* Set byte pattern for source block. */ 319 srand(pattern_seed); 320 for (i = 0; i < SIZE_MB(2); i++) { 321 ((char *)src)[i] = (char) rand(); 322 } 323 324 dest = src - SIZE_MB(2); 325 326 void *new_ptr = mremap(src + SIZE_MB(1), SIZE_MB(1), SIZE_MB(1), 327 MREMAP_MAYMOVE | MREMAP_FIXED, dest + SIZE_MB(1)); 328 if (new_ptr == MAP_FAILED) { 329 perror("mremap"); 330 success = 0; 331 goto out; 332 } 333 334 /* Verify byte pattern after remapping */ 335 srand(pattern_seed); 336 for (i = 0; i < SIZE_MB(1); i++) { 337 char c = (char) rand(); 338 339 if (((char *)src)[i] != c) { 340 ksft_print_msg("Data at src at %d got corrupted due to unrelated mremap\n", 341 i); 342 ksft_print_msg("Expected: %#x\t Got: %#x\n", c & 0xff, 343 ((char *) src)[i] & 0xff); 344 success = 0; 345 } 346 } 347 348 out: 349 if (munmap(ptr, size) == -1) 350 perror("munmap"); 351 352 if (success) 353 ksft_test_result_pass("%s\n", test_name); 354 else 355 ksft_test_result_fail("%s\n", test_name); 356 } 357 358 /* Returns the time taken for the remap on success else returns -1. */ 359 static long long remap_region(struct config c, unsigned int threshold_mb, 360 char pattern_seed) 361 { 362 void *addr, *src_addr, *dest_addr, *dest_preamble_addr; 363 unsigned long long i; 364 struct timespec t_start = {0, 0}, t_end = {0, 0}; 365 long long start_ns, end_ns, align_mask, ret, offset; 366 unsigned long long threshold; 367 368 if (threshold_mb == VALIDATION_NO_THRESHOLD) 369 threshold = c.region_size; 370 else 371 threshold = MIN(threshold_mb * _1MB, c.region_size); 372 373 src_addr = get_source_mapping(c); 374 if (!src_addr) { 375 ret = -1; 376 goto out; 377 } 378 379 /* Set byte pattern for source block. */ 380 srand(pattern_seed); 381 for (i = 0; i < threshold; i++) 382 memset((char *) src_addr + i, (char) rand(), 1); 383 384 /* Mask to zero out lower bits of address for alignment */ 385 align_mask = ~(c.dest_alignment - 1); 386 /* Offset of destination address from the end of the source region */ 387 offset = (c.overlapping) ? -c.dest_alignment : c.dest_alignment; 388 addr = (void *) (((unsigned long long) src_addr + c.region_size 389 + offset) & align_mask); 390 391 /* Remap after the destination block preamble. */ 392 addr += c.dest_preamble_size; 393 394 /* See comment in get_source_mapping() */ 395 if (!((unsigned long long) addr & c.dest_alignment)) 396 addr = (void *) ((unsigned long long) addr | c.dest_alignment); 397 398 /* Don't destroy existing mappings unless expected to overlap */ 399 while (!is_remap_region_valid(addr, c.region_size) && !c.overlapping) { 400 /* Check for unsigned overflow */ 401 if (addr + c.dest_alignment < addr) { 402 ksft_print_msg("Couldn't find a valid region to remap to\n"); 403 ret = -1; 404 goto clean_up_src; 405 } 406 addr += c.dest_alignment; 407 } 408 409 if (c.dest_preamble_size) { 410 dest_preamble_addr = mmap((void *) addr - c.dest_preamble_size, c.dest_preamble_size, 411 PROT_READ | PROT_WRITE, 412 MAP_FIXED_NOREPLACE | MAP_ANONYMOUS | MAP_SHARED, 413 -1, 0); 414 if (dest_preamble_addr == MAP_FAILED) { 415 ksft_print_msg("Failed to map dest preamble region: %s\n", 416 strerror(errno)); 417 ret = -1; 418 goto clean_up_src; 419 } 420 421 /* Set byte pattern for the dest preamble block. */ 422 srand(pattern_seed); 423 for (i = 0; i < c.dest_preamble_size; i++) 424 memset((char *) dest_preamble_addr + i, (char) rand(), 1); 425 } 426 427 clock_gettime(CLOCK_MONOTONIC, &t_start); 428 dest_addr = mremap(src_addr, c.region_size, c.region_size, 429 MREMAP_MAYMOVE|MREMAP_FIXED, (char *) addr); 430 clock_gettime(CLOCK_MONOTONIC, &t_end); 431 432 if (dest_addr == MAP_FAILED) { 433 ksft_print_msg("mremap failed: %s\n", strerror(errno)); 434 ret = -1; 435 goto clean_up_dest_preamble; 436 } 437 438 /* Verify byte pattern after remapping */ 439 srand(pattern_seed); 440 for (i = 0; i < threshold; i++) { 441 char c = (char) rand(); 442 443 if (((char *) dest_addr)[i] != c) { 444 ksft_print_msg("Data after remap doesn't match at offset %llu\n", 445 i); 446 ksft_print_msg("Expected: %#x\t Got: %#x\n", c & 0xff, 447 ((char *) dest_addr)[i] & 0xff); 448 ret = -1; 449 goto clean_up_dest; 450 } 451 } 452 453 /* Verify the dest preamble byte pattern after remapping */ 454 if (c.dest_preamble_size) { 455 srand(pattern_seed); 456 for (i = 0; i < c.dest_preamble_size; i++) { 457 char c = (char) rand(); 458 459 if (((char *) dest_preamble_addr)[i] != c) { 460 ksft_print_msg("Preamble data after remap doesn't match at offset %d\n", 461 i); 462 ksft_print_msg("Expected: %#x\t Got: %#x\n", c & 0xff, 463 ((char *) dest_preamble_addr)[i] & 0xff); 464 ret = -1; 465 goto clean_up_dest; 466 } 467 } 468 } 469 470 start_ns = t_start.tv_sec * NS_PER_SEC + t_start.tv_nsec; 471 end_ns = t_end.tv_sec * NS_PER_SEC + t_end.tv_nsec; 472 ret = end_ns - start_ns; 473 474 /* 475 * Since the destination address is specified using MREMAP_FIXED, subsequent 476 * mremap will unmap any previous mapping at the address range specified by 477 * dest_addr and region_size. This significantly affects the remap time of 478 * subsequent tests. So we clean up mappings after each test. 479 */ 480 clean_up_dest: 481 munmap(dest_addr, c.region_size); 482 clean_up_dest_preamble: 483 if (c.dest_preamble_size && dest_preamble_addr) 484 munmap(dest_preamble_addr, c.dest_preamble_size); 485 clean_up_src: 486 munmap(src_addr, c.region_size); 487 out: 488 return ret; 489 } 490 491 /* 492 * Verify that an mremap aligning down does not destroy 493 * the beginning of the mapping just because the aligned 494 * down address landed on a mapping that maybe does not exist. 495 */ 496 static void mremap_move_1mb_from_start(char pattern_seed) 497 { 498 char *test_name = "mremap move 1mb from start at 1MB+256KB aligned src"; 499 void *src = NULL, *dest = NULL; 500 int i, success = 1; 501 502 /* Config to reuse get_source_mapping() to do an aligned mmap. */ 503 struct config c = { 504 .src_alignment = SIZE_MB(1) + SIZE_KB(256), 505 .region_size = SIZE_MB(6) 506 }; 507 508 src = get_source_mapping(c); 509 if (!src) { 510 success = 0; 511 goto out; 512 } 513 514 c.src_alignment = SIZE_MB(1) + SIZE_KB(256); 515 dest = get_source_mapping(c); 516 if (!dest) { 517 success = 0; 518 goto out; 519 } 520 521 /* Set byte pattern for source block. */ 522 srand(pattern_seed); 523 for (i = 0; i < SIZE_MB(2); i++) { 524 ((char *)src)[i] = (char) rand(); 525 } 526 527 /* 528 * Unmap the beginning of dest so that the aligned address 529 * falls on no mapping. 530 */ 531 munmap(dest, SIZE_MB(1)); 532 533 void *new_ptr = mremap(src + SIZE_MB(1), SIZE_MB(1), SIZE_MB(1), 534 MREMAP_MAYMOVE | MREMAP_FIXED, dest + SIZE_MB(1)); 535 if (new_ptr == MAP_FAILED) { 536 perror("mremap"); 537 success = 0; 538 goto out; 539 } 540 541 /* Verify byte pattern after remapping */ 542 srand(pattern_seed); 543 for (i = 0; i < SIZE_MB(1); i++) { 544 char c = (char) rand(); 545 546 if (((char *)src)[i] != c) { 547 ksft_print_msg("Data at src at %d got corrupted due to unrelated mremap\n", 548 i); 549 ksft_print_msg("Expected: %#x\t Got: %#x\n", c & 0xff, 550 ((char *) src)[i] & 0xff); 551 success = 0; 552 } 553 } 554 555 out: 556 if (src && munmap(src, c.region_size) == -1) 557 perror("munmap src"); 558 559 if (dest && munmap(dest, c.region_size) == -1) 560 perror("munmap dest"); 561 562 if (success) 563 ksft_test_result_pass("%s\n", test_name); 564 else 565 ksft_test_result_fail("%s\n", test_name); 566 } 567 568 static void run_mremap_test_case(struct test test_case, int *failures, 569 unsigned int threshold_mb, 570 unsigned int pattern_seed) 571 { 572 long long remap_time = remap_region(test_case.config, threshold_mb, 573 pattern_seed); 574 575 if (remap_time < 0) { 576 if (test_case.expect_failure) 577 ksft_test_result_xfail("%s\n\tExpected mremap failure\n", 578 test_case.name); 579 else { 580 ksft_test_result_fail("%s\n", test_case.name); 581 *failures += 1; 582 } 583 } else { 584 /* 585 * Comparing mremap time is only applicable if entire region 586 * was faulted in. 587 */ 588 if (threshold_mb == VALIDATION_NO_THRESHOLD || 589 test_case.config.region_size <= threshold_mb * _1MB) 590 ksft_test_result_pass("%s\n\tmremap time: %12lldns\n", 591 test_case.name, remap_time); 592 else 593 ksft_test_result_pass("%s\n", test_case.name); 594 } 595 } 596 597 static void usage(const char *cmd) 598 { 599 fprintf(stderr, 600 "Usage: %s [[-t <threshold_mb>] [-p <pattern_seed>]]\n" 601 "-t\t only validate threshold_mb of the remapped region\n" 602 " \t if 0 is supplied no threshold is used; all tests\n" 603 " \t are run and remapped regions validated fully.\n" 604 " \t The default threshold used is 4MB.\n" 605 "-p\t provide a seed to generate the random pattern for\n" 606 " \t validating the remapped region.\n", cmd); 607 } 608 609 static int parse_args(int argc, char **argv, unsigned int *threshold_mb, 610 unsigned int *pattern_seed) 611 { 612 const char *optstr = "t:p:"; 613 int opt; 614 615 while ((opt = getopt(argc, argv, optstr)) != -1) { 616 switch (opt) { 617 case 't': 618 *threshold_mb = atoi(optarg); 619 break; 620 case 'p': 621 *pattern_seed = atoi(optarg); 622 break; 623 default: 624 usage(argv[0]); 625 return -1; 626 } 627 } 628 629 if (optind < argc) { 630 usage(argv[0]); 631 return -1; 632 } 633 634 return 0; 635 } 636 637 #define MAX_TEST 15 638 #define MAX_PERF_TEST 3 639 int main(int argc, char **argv) 640 { 641 int failures = 0; 642 int i, run_perf_tests; 643 unsigned int threshold_mb = VALIDATION_DEFAULT_THRESHOLD; 644 unsigned int pattern_seed; 645 int num_expand_tests = 2; 646 int num_misc_tests = 2; 647 struct test test_cases[MAX_TEST] = {}; 648 struct test perf_test_cases[MAX_PERF_TEST]; 649 int page_size; 650 time_t t; 651 FILE *maps_fp; 652 653 pattern_seed = (unsigned int) time(&t); 654 655 if (parse_args(argc, argv, &threshold_mb, &pattern_seed) < 0) 656 exit(EXIT_FAILURE); 657 658 ksft_print_msg("Test configs:\n\tthreshold_mb=%u\n\tpattern_seed=%u\n\n", 659 threshold_mb, pattern_seed); 660 661 page_size = sysconf(_SC_PAGESIZE); 662 663 /* Expected mremap failures */ 664 test_cases[0] = MAKE_TEST(page_size, page_size, page_size, 665 OVERLAPPING, EXPECT_FAILURE, 666 "mremap - Source and Destination Regions Overlapping"); 667 668 test_cases[1] = MAKE_TEST(page_size, page_size/4, page_size, 669 NON_OVERLAPPING, EXPECT_FAILURE, 670 "mremap - Destination Address Misaligned (1KB-aligned)"); 671 test_cases[2] = MAKE_TEST(page_size/4, page_size, page_size, 672 NON_OVERLAPPING, EXPECT_FAILURE, 673 "mremap - Source Address Misaligned (1KB-aligned)"); 674 675 /* Src addr PTE aligned */ 676 test_cases[3] = MAKE_TEST(PTE, PTE, PTE * 2, 677 NON_OVERLAPPING, EXPECT_SUCCESS, 678 "8KB mremap - Source PTE-aligned, Destination PTE-aligned"); 679 680 /* Src addr 1MB aligned */ 681 test_cases[4] = MAKE_TEST(_1MB, PTE, _2MB, NON_OVERLAPPING, EXPECT_SUCCESS, 682 "2MB mremap - Source 1MB-aligned, Destination PTE-aligned"); 683 test_cases[5] = MAKE_TEST(_1MB, _1MB, _2MB, NON_OVERLAPPING, EXPECT_SUCCESS, 684 "2MB mremap - Source 1MB-aligned, Destination 1MB-aligned"); 685 686 /* Src addr PMD aligned */ 687 test_cases[6] = MAKE_TEST(PMD, PTE, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS, 688 "4MB mremap - Source PMD-aligned, Destination PTE-aligned"); 689 test_cases[7] = MAKE_TEST(PMD, _1MB, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS, 690 "4MB mremap - Source PMD-aligned, Destination 1MB-aligned"); 691 test_cases[8] = MAKE_TEST(PMD, PMD, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS, 692 "4MB mremap - Source PMD-aligned, Destination PMD-aligned"); 693 694 /* Src addr PUD aligned */ 695 test_cases[9] = MAKE_TEST(PUD, PTE, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS, 696 "2GB mremap - Source PUD-aligned, Destination PTE-aligned"); 697 test_cases[10] = MAKE_TEST(PUD, _1MB, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS, 698 "2GB mremap - Source PUD-aligned, Destination 1MB-aligned"); 699 test_cases[11] = MAKE_TEST(PUD, PMD, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS, 700 "2GB mremap - Source PUD-aligned, Destination PMD-aligned"); 701 test_cases[12] = MAKE_TEST(PUD, PUD, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS, 702 "2GB mremap - Source PUD-aligned, Destination PUD-aligned"); 703 704 /* Src and Dest addr 1MB aligned. 5MB mremap. */ 705 test_cases[13] = MAKE_TEST(_1MB, _1MB, _5MB, NON_OVERLAPPING, EXPECT_SUCCESS, 706 "5MB mremap - Source 1MB-aligned, Destination 1MB-aligned"); 707 708 /* Src and Dest addr 1MB aligned. 5MB mremap. */ 709 test_cases[14] = MAKE_TEST(_1MB, _1MB, _5MB, NON_OVERLAPPING, EXPECT_SUCCESS, 710 "5MB mremap - Source 1MB-aligned, Dest 1MB-aligned with 40MB Preamble"); 711 test_cases[14].config.dest_preamble_size = 10 * _4MB; 712 713 perf_test_cases[0] = MAKE_TEST(page_size, page_size, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS, 714 "1GB mremap - Source PTE-aligned, Destination PTE-aligned"); 715 /* 716 * mremap 1GB region - Page table level aligned time 717 * comparison. 718 */ 719 perf_test_cases[1] = MAKE_TEST(PMD, PMD, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS, 720 "1GB mremap - Source PMD-aligned, Destination PMD-aligned"); 721 perf_test_cases[2] = MAKE_TEST(PUD, PUD, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS, 722 "1GB mremap - Source PUD-aligned, Destination PUD-aligned"); 723 724 run_perf_tests = (threshold_mb == VALIDATION_NO_THRESHOLD) || 725 (threshold_mb * _1MB >= _1GB); 726 727 ksft_set_plan(ARRAY_SIZE(test_cases) + (run_perf_tests ? 728 ARRAY_SIZE(perf_test_cases) : 0) + num_expand_tests + num_misc_tests); 729 730 for (i = 0; i < ARRAY_SIZE(test_cases); i++) 731 run_mremap_test_case(test_cases[i], &failures, threshold_mb, 732 pattern_seed); 733 734 maps_fp = fopen("/proc/self/maps", "r"); 735 736 if (maps_fp == NULL) { 737 ksft_print_msg("Failed to read /proc/self/maps: %s\n", strerror(errno)); 738 exit(KSFT_FAIL); 739 } 740 741 mremap_expand_merge(maps_fp, page_size); 742 mremap_expand_merge_offset(maps_fp, page_size); 743 744 fclose(maps_fp); 745 746 mremap_move_within_range(pattern_seed); 747 mremap_move_1mb_from_start(pattern_seed); 748 749 if (run_perf_tests) { 750 ksft_print_msg("\n%s\n", 751 "mremap HAVE_MOVE_PMD/PUD optimization time comparison for 1GB region:"); 752 for (i = 0; i < ARRAY_SIZE(perf_test_cases); i++) 753 run_mremap_test_case(perf_test_cases[i], &failures, 754 threshold_mb, pattern_seed); 755 } 756 757 if (failures > 0) 758 ksft_exit_fail(); 759 else 760 ksft_exit_pass(); 761 } 762