1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright 2020 Google LLC
4 */
5 #define _GNU_SOURCE
6
7 #include <errno.h>
8 #include <stdlib.h>
9 #include <stdio.h>
10 #include <string.h>
11 #include <sys/mman.h>
12 #include <time.h>
13 #include <stdbool.h>
14
15 #include "../kselftest.h"
16
17 #define EXPECT_SUCCESS 0
18 #define EXPECT_FAILURE 1
19 #define NON_OVERLAPPING 0
20 #define OVERLAPPING 1
21 #define NS_PER_SEC 1000000000ULL
22 #define VALIDATION_DEFAULT_THRESHOLD 4 /* 4MB */
23 #define VALIDATION_NO_THRESHOLD 0 /* Verify the entire region */
24
25 #ifndef MIN
26 #define MIN(X, Y) ((X) < (Y) ? (X) : (Y))
27 #define MAX(X, Y) ((X) > (Y) ? (X) : (Y))
28 #endif
29 #define SIZE_MB(m) ((size_t)m * (1024 * 1024))
30 #define SIZE_KB(k) ((size_t)k * 1024)
31
32 struct config {
33 unsigned long long src_alignment;
34 unsigned long long dest_alignment;
35 unsigned long long region_size;
36 int overlapping;
37 int dest_preamble_size;
38 };
39
40 struct test {
41 const char *name;
42 struct config config;
43 int expect_failure;
44 };
45
46 enum {
47 _1KB = 1ULL << 10, /* 1KB -> not page aligned */
48 _4KB = 4ULL << 10,
49 _8KB = 8ULL << 10,
50 _1MB = 1ULL << 20,
51 _2MB = 2ULL << 20,
52 _4MB = 4ULL << 20,
53 _5MB = 5ULL << 20,
54 _1GB = 1ULL << 30,
55 _2GB = 2ULL << 30,
56 PMD = _2MB,
57 PUD = _1GB,
58 };
59
60 #define PTE page_size
61
62 #define MAKE_TEST(source_align, destination_align, size, \
63 overlaps, should_fail, test_name) \
64 (struct test){ \
65 .name = test_name, \
66 .config = { \
67 .src_alignment = source_align, \
68 .dest_alignment = destination_align, \
69 .region_size = size, \
70 .overlapping = overlaps, \
71 }, \
72 .expect_failure = should_fail \
73 }
74
75 /* compute square root using binary search */
get_sqrt(unsigned long val)76 static unsigned long get_sqrt(unsigned long val)
77 {
78 unsigned long low = 1;
79
80 /* assuming rand_size is less than 1TB */
81 unsigned long high = (1UL << 20);
82
83 while (low <= high) {
84 unsigned long mid = low + (high - low) / 2;
85 unsigned long temp = mid * mid;
86
87 if (temp == val)
88 return mid;
89 if (temp < val)
90 low = mid + 1;
91 high = mid - 1;
92 }
93 return low;
94 }
95
96 /*
97 * Returns false if the requested remap region overlaps with an
98 * existing mapping (e.g text, stack) else returns true.
99 */
is_remap_region_valid(void * addr,unsigned long long size)100 static bool is_remap_region_valid(void *addr, unsigned long long size)
101 {
102 void *remap_addr = NULL;
103 bool ret = true;
104
105 /* Use MAP_FIXED_NOREPLACE flag to ensure region is not mapped */
106 remap_addr = mmap(addr, size, PROT_READ | PROT_WRITE,
107 MAP_FIXED_NOREPLACE | MAP_ANONYMOUS | MAP_SHARED,
108 -1, 0);
109
110 if (remap_addr == MAP_FAILED) {
111 if (errno == EEXIST)
112 ret = false;
113 } else {
114 munmap(remap_addr, size);
115 }
116
117 return ret;
118 }
119
120 /* Returns mmap_min_addr sysctl tunable from procfs */
get_mmap_min_addr(void)121 static unsigned long long get_mmap_min_addr(void)
122 {
123 FILE *fp;
124 int n_matched;
125 static unsigned long long addr;
126
127 if (addr)
128 return addr;
129
130 fp = fopen("/proc/sys/vm/mmap_min_addr", "r");
131 if (fp == NULL) {
132 ksft_print_msg("Failed to open /proc/sys/vm/mmap_min_addr: %s\n",
133 strerror(errno));
134 exit(KSFT_SKIP);
135 }
136
137 n_matched = fscanf(fp, "%llu", &addr);
138 if (n_matched != 1) {
139 ksft_print_msg("Failed to read /proc/sys/vm/mmap_min_addr: %s\n",
140 strerror(errno));
141 fclose(fp);
142 exit(KSFT_SKIP);
143 }
144
145 fclose(fp);
146 return addr;
147 }
148
149 /*
150 * Using /proc/self/maps, assert that the specified address range is contained
151 * within a single mapping.
152 */
is_range_mapped(FILE * maps_fp,unsigned long start,unsigned long end)153 static bool is_range_mapped(FILE *maps_fp, unsigned long start,
154 unsigned long end)
155 {
156 char *line = NULL;
157 size_t len = 0;
158 bool success = false;
159 unsigned long first_val, second_val;
160
161 rewind(maps_fp);
162
163 while (getline(&line, &len, maps_fp) != -1) {
164 if (sscanf(line, "%lx-%lx", &first_val, &second_val) != 2) {
165 ksft_exit_fail_msg("cannot parse /proc/self/maps\n");
166 break;
167 }
168
169 if (first_val <= start && second_val >= end) {
170 success = true;
171 break;
172 }
173 }
174
175 return success;
176 }
177
178 /*
179 * Returns the start address of the mapping on success, else returns
180 * NULL on failure.
181 */
get_source_mapping(struct config c)182 static void *get_source_mapping(struct config c)
183 {
184 unsigned long long addr = 0ULL;
185 void *src_addr = NULL;
186 unsigned long long mmap_min_addr;
187
188 mmap_min_addr = get_mmap_min_addr();
189 /*
190 * For some tests, we need to not have any mappings below the
191 * source mapping. Add some headroom to mmap_min_addr for this.
192 */
193 mmap_min_addr += 10 * _4MB;
194
195 retry:
196 addr += c.src_alignment;
197 if (addr < mmap_min_addr)
198 goto retry;
199
200 src_addr = mmap((void *) addr, c.region_size, PROT_READ | PROT_WRITE,
201 MAP_FIXED_NOREPLACE | MAP_ANONYMOUS | MAP_SHARED,
202 -1, 0);
203 if (src_addr == MAP_FAILED) {
204 if (errno == EPERM || errno == EEXIST)
205 goto retry;
206 goto error;
207 }
208 /*
209 * Check that the address is aligned to the specified alignment.
210 * Addresses which have alignments that are multiples of that
211 * specified are not considered valid. For instance, 1GB address is
212 * 2MB-aligned, however it will not be considered valid for a
213 * requested alignment of 2MB. This is done to reduce coincidental
214 * alignment in the tests.
215 */
216 if (((unsigned long long) src_addr & (c.src_alignment - 1)) ||
217 !((unsigned long long) src_addr & c.src_alignment)) {
218 munmap(src_addr, c.region_size);
219 goto retry;
220 }
221
222 if (!src_addr)
223 goto error;
224
225 return src_addr;
226 error:
227 ksft_print_msg("Failed to map source region: %s\n",
228 strerror(errno));
229 return NULL;
230 }
231
232 /*
233 * This test validates that merge is called when expanding a mapping.
234 * Mapping containing three pages is created, middle page is unmapped
235 * and then the mapping containing the first page is expanded so that
236 * it fills the created hole. The two parts should merge creating
237 * single mapping with three pages.
238 */
mremap_expand_merge(FILE * maps_fp,unsigned long page_size)239 static void mremap_expand_merge(FILE *maps_fp, unsigned long page_size)
240 {
241 char *test_name = "mremap expand merge";
242 bool success = false;
243 char *remap, *start;
244
245 start = mmap(NULL, 3 * page_size, PROT_READ | PROT_WRITE,
246 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
247
248 if (start == MAP_FAILED) {
249 ksft_print_msg("mmap failed: %s\n", strerror(errno));
250 goto out;
251 }
252
253 munmap(start + page_size, page_size);
254 remap = mremap(start, page_size, 2 * page_size, 0);
255 if (remap == MAP_FAILED) {
256 ksft_print_msg("mremap failed: %s\n", strerror(errno));
257 munmap(start, page_size);
258 munmap(start + 2 * page_size, page_size);
259 goto out;
260 }
261
262 success = is_range_mapped(maps_fp, (unsigned long)start,
263 (unsigned long)(start + 3 * page_size));
264 munmap(start, 3 * page_size);
265
266 out:
267 if (success)
268 ksft_test_result_pass("%s\n", test_name);
269 else
270 ksft_test_result_fail("%s\n", test_name);
271 }
272
273 /*
274 * Similar to mremap_expand_merge() except instead of removing the middle page,
275 * we remove the last then attempt to remap offset from the second page. This
276 * should result in the mapping being restored to its former state.
277 */
mremap_expand_merge_offset(FILE * maps_fp,unsigned long page_size)278 static void mremap_expand_merge_offset(FILE *maps_fp, unsigned long page_size)
279 {
280
281 char *test_name = "mremap expand merge offset";
282 bool success = false;
283 char *remap, *start;
284
285 start = mmap(NULL, 3 * page_size, PROT_READ | PROT_WRITE,
286 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
287
288 if (start == MAP_FAILED) {
289 ksft_print_msg("mmap failed: %s\n", strerror(errno));
290 goto out;
291 }
292
293 /* Unmap final page to ensure we have space to expand. */
294 munmap(start + 2 * page_size, page_size);
295 remap = mremap(start + page_size, page_size, 2 * page_size, 0);
296 if (remap == MAP_FAILED) {
297 ksft_print_msg("mremap failed: %s\n", strerror(errno));
298 munmap(start, 2 * page_size);
299 goto out;
300 }
301
302 success = is_range_mapped(maps_fp, (unsigned long)start,
303 (unsigned long)(start + 3 * page_size));
304 munmap(start, 3 * page_size);
305
306 out:
307 if (success)
308 ksft_test_result_pass("%s\n", test_name);
309 else
310 ksft_test_result_fail("%s\n", test_name);
311 }
312
313 /*
314 * Verify that an mremap within a range does not cause corruption
315 * of unrelated part of range.
316 *
317 * Consider the following range which is 2MB aligned and is
318 * a part of a larger 20MB range which is not shown. Each
319 * character is 256KB below making the source and destination
320 * 2MB each. The lower case letters are moved (s to d) and the
321 * upper case letters are not moved. The below test verifies
322 * that the upper case S letters are not corrupted by the
323 * adjacent mremap.
324 *
325 * |DDDDddddSSSSssss|
326 */
mremap_move_within_range(unsigned int pattern_seed,char * rand_addr)327 static void mremap_move_within_range(unsigned int pattern_seed, char *rand_addr)
328 {
329 char *test_name = "mremap mremap move within range";
330 void *src, *dest;
331 int i, success = 1;
332
333 size_t size = SIZE_MB(20);
334 void *ptr = mmap(NULL, size, PROT_READ | PROT_WRITE,
335 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
336 if (ptr == MAP_FAILED) {
337 perror("mmap");
338 success = 0;
339 goto out;
340 }
341 memset(ptr, 0, size);
342
343 src = ptr + SIZE_MB(6);
344 src = (void *)((unsigned long)src & ~(SIZE_MB(2) - 1));
345
346 /* Set byte pattern for source block. */
347 memcpy(src, rand_addr, SIZE_MB(2));
348
349 dest = src - SIZE_MB(2);
350
351 void *new_ptr = mremap(src + SIZE_MB(1), SIZE_MB(1), SIZE_MB(1),
352 MREMAP_MAYMOVE | MREMAP_FIXED, dest + SIZE_MB(1));
353 if (new_ptr == MAP_FAILED) {
354 perror("mremap");
355 success = 0;
356 goto out;
357 }
358
359 /* Verify byte pattern after remapping */
360 srand(pattern_seed);
361 for (i = 0; i < SIZE_MB(1); i++) {
362 char c = (char) rand();
363
364 if (((char *)src)[i] != c) {
365 ksft_print_msg("Data at src at %d got corrupted due to unrelated mremap\n",
366 i);
367 ksft_print_msg("Expected: %#x\t Got: %#x\n", c & 0xff,
368 ((char *) src)[i] & 0xff);
369 success = 0;
370 }
371 }
372
373 out:
374 if (munmap(ptr, size) == -1)
375 perror("munmap");
376
377 if (success)
378 ksft_test_result_pass("%s\n", test_name);
379 else
380 ksft_test_result_fail("%s\n", test_name);
381 }
382
383 /* Returns the time taken for the remap on success else returns -1. */
remap_region(struct config c,unsigned int threshold_mb,char * rand_addr)384 static long long remap_region(struct config c, unsigned int threshold_mb,
385 char *rand_addr)
386 {
387 void *addr, *src_addr, *dest_addr, *dest_preamble_addr;
388 unsigned long long t, d;
389 struct timespec t_start = {0, 0}, t_end = {0, 0};
390 long long start_ns, end_ns, align_mask, ret, offset;
391 unsigned long long threshold;
392 unsigned long num_chunks;
393
394 if (threshold_mb == VALIDATION_NO_THRESHOLD)
395 threshold = c.region_size;
396 else
397 threshold = MIN(threshold_mb * _1MB, c.region_size);
398
399 src_addr = get_source_mapping(c);
400 if (!src_addr) {
401 ret = -1;
402 goto out;
403 }
404
405 /* Set byte pattern for source block. */
406 memcpy(src_addr, rand_addr, threshold);
407
408 /* Mask to zero out lower bits of address for alignment */
409 align_mask = ~(c.dest_alignment - 1);
410 /* Offset of destination address from the end of the source region */
411 offset = (c.overlapping) ? -c.dest_alignment : c.dest_alignment;
412 addr = (void *) (((unsigned long long) src_addr + c.region_size
413 + offset) & align_mask);
414
415 /* Remap after the destination block preamble. */
416 addr += c.dest_preamble_size;
417
418 /* See comment in get_source_mapping() */
419 if (!((unsigned long long) addr & c.dest_alignment))
420 addr = (void *) ((unsigned long long) addr | c.dest_alignment);
421
422 /* Don't destroy existing mappings unless expected to overlap */
423 while (!is_remap_region_valid(addr, c.region_size) && !c.overlapping) {
424 /* Check for unsigned overflow */
425 if (addr + c.dest_alignment < addr) {
426 ksft_print_msg("Couldn't find a valid region to remap to\n");
427 ret = -1;
428 goto clean_up_src;
429 }
430 addr += c.dest_alignment;
431 }
432
433 if (c.dest_preamble_size) {
434 dest_preamble_addr = mmap((void *) addr - c.dest_preamble_size, c.dest_preamble_size,
435 PROT_READ | PROT_WRITE,
436 MAP_FIXED_NOREPLACE | MAP_ANONYMOUS | MAP_SHARED,
437 -1, 0);
438 if (dest_preamble_addr == MAP_FAILED) {
439 ksft_print_msg("Failed to map dest preamble region: %s\n",
440 strerror(errno));
441 ret = -1;
442 goto clean_up_src;
443 }
444
445 /* Set byte pattern for the dest preamble block. */
446 memcpy(dest_preamble_addr, rand_addr, c.dest_preamble_size);
447 }
448
449 clock_gettime(CLOCK_MONOTONIC, &t_start);
450 dest_addr = mremap(src_addr, c.region_size, c.region_size,
451 MREMAP_MAYMOVE|MREMAP_FIXED, (char *) addr);
452 clock_gettime(CLOCK_MONOTONIC, &t_end);
453
454 if (dest_addr == MAP_FAILED) {
455 ksft_print_msg("mremap failed: %s\n", strerror(errno));
456 ret = -1;
457 goto clean_up_dest_preamble;
458 }
459
460 /*
461 * Verify byte pattern after remapping. Employ an algorithm with a
462 * square root time complexity in threshold: divide the range into
463 * chunks, if memcmp() returns non-zero, only then perform an
464 * iteration in that chunk to find the mismatch index.
465 */
466 num_chunks = get_sqrt(threshold);
467 for (unsigned long i = 0; i < num_chunks; ++i) {
468 size_t chunk_size = threshold / num_chunks;
469 unsigned long shift = i * chunk_size;
470
471 if (!memcmp(dest_addr + shift, rand_addr + shift, chunk_size))
472 continue;
473
474 /* brute force iteration only over mismatch segment */
475 for (t = shift; t < shift + chunk_size; ++t) {
476 if (((char *) dest_addr)[t] != rand_addr[t]) {
477 ksft_print_msg("Data after remap doesn't match at offset %llu\n",
478 t);
479 ksft_print_msg("Expected: %#x\t Got: %#x\n", rand_addr[t] & 0xff,
480 ((char *) dest_addr)[t] & 0xff);
481 ret = -1;
482 goto clean_up_dest;
483 }
484 }
485 }
486
487 /*
488 * if threshold is not divisible by num_chunks, then check the
489 * last chunk
490 */
491 for (t = num_chunks * (threshold / num_chunks); t < threshold; ++t) {
492 if (((char *) dest_addr)[t] != rand_addr[t]) {
493 ksft_print_msg("Data after remap doesn't match at offset %llu\n",
494 t);
495 ksft_print_msg("Expected: %#x\t Got: %#x\n", rand_addr[t] & 0xff,
496 ((char *) dest_addr)[t] & 0xff);
497 ret = -1;
498 goto clean_up_dest;
499 }
500 }
501
502 /* Verify the dest preamble byte pattern after remapping */
503 if (!c.dest_preamble_size)
504 goto no_preamble;
505
506 num_chunks = get_sqrt(c.dest_preamble_size);
507
508 for (unsigned long i = 0; i < num_chunks; ++i) {
509 size_t chunk_size = c.dest_preamble_size / num_chunks;
510 unsigned long shift = i * chunk_size;
511
512 if (!memcmp(dest_preamble_addr + shift, rand_addr + shift,
513 chunk_size))
514 continue;
515
516 /* brute force iteration only over mismatched segment */
517 for (d = shift; d < shift + chunk_size; ++d) {
518 if (((char *) dest_preamble_addr)[d] != rand_addr[d]) {
519 ksft_print_msg("Preamble data after remap doesn't match at offset %llu\n",
520 d);
521 ksft_print_msg("Expected: %#x\t Got: %#x\n", rand_addr[d] & 0xff,
522 ((char *) dest_preamble_addr)[d] & 0xff);
523 ret = -1;
524 goto clean_up_dest;
525 }
526 }
527 }
528
529 for (d = num_chunks * (c.dest_preamble_size / num_chunks); d < c.dest_preamble_size; ++d) {
530 if (((char *) dest_preamble_addr)[d] != rand_addr[d]) {
531 ksft_print_msg("Preamble data after remap doesn't match at offset %llu\n",
532 d);
533 ksft_print_msg("Expected: %#x\t Got: %#x\n", rand_addr[d] & 0xff,
534 ((char *) dest_preamble_addr)[d] & 0xff);
535 ret = -1;
536 goto clean_up_dest;
537 }
538 }
539
540 no_preamble:
541 start_ns = t_start.tv_sec * NS_PER_SEC + t_start.tv_nsec;
542 end_ns = t_end.tv_sec * NS_PER_SEC + t_end.tv_nsec;
543 ret = end_ns - start_ns;
544
545 /*
546 * Since the destination address is specified using MREMAP_FIXED, subsequent
547 * mremap will unmap any previous mapping at the address range specified by
548 * dest_addr and region_size. This significantly affects the remap time of
549 * subsequent tests. So we clean up mappings after each test.
550 */
551 clean_up_dest:
552 munmap(dest_addr, c.region_size);
553 clean_up_dest_preamble:
554 if (c.dest_preamble_size && dest_preamble_addr)
555 munmap(dest_preamble_addr, c.dest_preamble_size);
556 clean_up_src:
557 munmap(src_addr, c.region_size);
558 out:
559 return ret;
560 }
561
562 /*
563 * Verify that an mremap aligning down does not destroy
564 * the beginning of the mapping just because the aligned
565 * down address landed on a mapping that maybe does not exist.
566 */
mremap_move_1mb_from_start(unsigned int pattern_seed,char * rand_addr)567 static void mremap_move_1mb_from_start(unsigned int pattern_seed,
568 char *rand_addr)
569 {
570 char *test_name = "mremap move 1mb from start at 1MB+256KB aligned src";
571 void *src = NULL, *dest = NULL;
572 int i, success = 1;
573
574 /* Config to reuse get_source_mapping() to do an aligned mmap. */
575 struct config c = {
576 .src_alignment = SIZE_MB(1) + SIZE_KB(256),
577 .region_size = SIZE_MB(6)
578 };
579
580 src = get_source_mapping(c);
581 if (!src) {
582 success = 0;
583 goto out;
584 }
585
586 c.src_alignment = SIZE_MB(1) + SIZE_KB(256);
587 dest = get_source_mapping(c);
588 if (!dest) {
589 success = 0;
590 goto out;
591 }
592
593 /* Set byte pattern for source block. */
594 memcpy(src, rand_addr, SIZE_MB(2));
595
596 /*
597 * Unmap the beginning of dest so that the aligned address
598 * falls on no mapping.
599 */
600 munmap(dest, SIZE_MB(1));
601
602 void *new_ptr = mremap(src + SIZE_MB(1), SIZE_MB(1), SIZE_MB(1),
603 MREMAP_MAYMOVE | MREMAP_FIXED, dest + SIZE_MB(1));
604 if (new_ptr == MAP_FAILED) {
605 perror("mremap");
606 success = 0;
607 goto out;
608 }
609
610 /* Verify byte pattern after remapping */
611 srand(pattern_seed);
612 for (i = 0; i < SIZE_MB(1); i++) {
613 char c = (char) rand();
614
615 if (((char *)src)[i] != c) {
616 ksft_print_msg("Data at src at %d got corrupted due to unrelated mremap\n",
617 i);
618 ksft_print_msg("Expected: %#x\t Got: %#x\n", c & 0xff,
619 ((char *) src)[i] & 0xff);
620 success = 0;
621 }
622 }
623
624 out:
625 if (src && munmap(src, c.region_size) == -1)
626 perror("munmap src");
627
628 if (dest && munmap(dest, c.region_size) == -1)
629 perror("munmap dest");
630
631 if (success)
632 ksft_test_result_pass("%s\n", test_name);
633 else
634 ksft_test_result_fail("%s\n", test_name);
635 }
636
run_mremap_test_case(struct test test_case,int * failures,unsigned int threshold_mb,unsigned int pattern_seed,char * rand_addr)637 static void run_mremap_test_case(struct test test_case, int *failures,
638 unsigned int threshold_mb,
639 unsigned int pattern_seed, char *rand_addr)
640 {
641 long long remap_time = remap_region(test_case.config, threshold_mb,
642 rand_addr);
643
644 if (remap_time < 0) {
645 if (test_case.expect_failure)
646 ksft_test_result_xfail("%s\n\tExpected mremap failure\n",
647 test_case.name);
648 else {
649 ksft_test_result_fail("%s\n", test_case.name);
650 *failures += 1;
651 }
652 } else {
653 /*
654 * Comparing mremap time is only applicable if entire region
655 * was faulted in.
656 */
657 if (threshold_mb == VALIDATION_NO_THRESHOLD ||
658 test_case.config.region_size <= threshold_mb * _1MB)
659 ksft_test_result_pass("%s\n\tmremap time: %12lldns\n",
660 test_case.name, remap_time);
661 else
662 ksft_test_result_pass("%s\n", test_case.name);
663 }
664 }
665
usage(const char * cmd)666 static void usage(const char *cmd)
667 {
668 fprintf(stderr,
669 "Usage: %s [[-t <threshold_mb>] [-p <pattern_seed>]]\n"
670 "-t\t only validate threshold_mb of the remapped region\n"
671 " \t if 0 is supplied no threshold is used; all tests\n"
672 " \t are run and remapped regions validated fully.\n"
673 " \t The default threshold used is 4MB.\n"
674 "-p\t provide a seed to generate the random pattern for\n"
675 " \t validating the remapped region.\n", cmd);
676 }
677
parse_args(int argc,char ** argv,unsigned int * threshold_mb,unsigned int * pattern_seed)678 static int parse_args(int argc, char **argv, unsigned int *threshold_mb,
679 unsigned int *pattern_seed)
680 {
681 const char *optstr = "t:p:";
682 int opt;
683
684 while ((opt = getopt(argc, argv, optstr)) != -1) {
685 switch (opt) {
686 case 't':
687 *threshold_mb = atoi(optarg);
688 break;
689 case 'p':
690 *pattern_seed = atoi(optarg);
691 break;
692 default:
693 usage(argv[0]);
694 return -1;
695 }
696 }
697
698 if (optind < argc) {
699 usage(argv[0]);
700 return -1;
701 }
702
703 return 0;
704 }
705
706 #define MAX_TEST 15
707 #define MAX_PERF_TEST 3
main(int argc,char ** argv)708 int main(int argc, char **argv)
709 {
710 int failures = 0;
711 int i, run_perf_tests;
712 unsigned int threshold_mb = VALIDATION_DEFAULT_THRESHOLD;
713
714 /* hard-coded test configs */
715 size_t max_test_variable_region_size = _2GB;
716 size_t max_test_constant_region_size = _2MB;
717 size_t dest_preamble_size = 10 * _4MB;
718
719 unsigned int pattern_seed;
720 char *rand_addr;
721 size_t rand_size;
722 int num_expand_tests = 2;
723 int num_misc_tests = 2;
724 struct test test_cases[MAX_TEST] = {};
725 struct test perf_test_cases[MAX_PERF_TEST];
726 int page_size;
727 time_t t;
728 FILE *maps_fp;
729
730 pattern_seed = (unsigned int) time(&t);
731
732 if (parse_args(argc, argv, &threshold_mb, &pattern_seed) < 0)
733 exit(EXIT_FAILURE);
734
735 ksft_print_msg("Test configs:\n\tthreshold_mb=%u\n\tpattern_seed=%u\n\n",
736 threshold_mb, pattern_seed);
737
738 /*
739 * set preallocated random array according to test configs; see the
740 * functions for the logic of setting the size
741 */
742 if (!threshold_mb)
743 rand_size = MAX(max_test_variable_region_size,
744 max_test_constant_region_size);
745 else
746 rand_size = MAX(MIN(threshold_mb * _1MB,
747 max_test_variable_region_size),
748 max_test_constant_region_size);
749 rand_size = MAX(dest_preamble_size, rand_size);
750
751 rand_addr = (char *)mmap(NULL, rand_size, PROT_READ | PROT_WRITE,
752 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
753 if (rand_addr == MAP_FAILED) {
754 perror("mmap");
755 ksft_exit_fail_msg("cannot mmap rand_addr\n");
756 }
757
758 /* fill stream of random bytes */
759 srand(pattern_seed);
760 for (unsigned long i = 0; i < rand_size; ++i)
761 rand_addr[i] = (char) rand();
762
763 page_size = sysconf(_SC_PAGESIZE);
764
765 /* Expected mremap failures */
766 test_cases[0] = MAKE_TEST(page_size, page_size, page_size,
767 OVERLAPPING, EXPECT_FAILURE,
768 "mremap - Source and Destination Regions Overlapping");
769
770 test_cases[1] = MAKE_TEST(page_size, page_size/4, page_size,
771 NON_OVERLAPPING, EXPECT_FAILURE,
772 "mremap - Destination Address Misaligned (1KB-aligned)");
773 test_cases[2] = MAKE_TEST(page_size/4, page_size, page_size,
774 NON_OVERLAPPING, EXPECT_FAILURE,
775 "mremap - Source Address Misaligned (1KB-aligned)");
776
777 /* Src addr PTE aligned */
778 test_cases[3] = MAKE_TEST(PTE, PTE, PTE * 2,
779 NON_OVERLAPPING, EXPECT_SUCCESS,
780 "8KB mremap - Source PTE-aligned, Destination PTE-aligned");
781
782 /* Src addr 1MB aligned */
783 test_cases[4] = MAKE_TEST(_1MB, PTE, _2MB, NON_OVERLAPPING, EXPECT_SUCCESS,
784 "2MB mremap - Source 1MB-aligned, Destination PTE-aligned");
785 test_cases[5] = MAKE_TEST(_1MB, _1MB, _2MB, NON_OVERLAPPING, EXPECT_SUCCESS,
786 "2MB mremap - Source 1MB-aligned, Destination 1MB-aligned");
787
788 /* Src addr PMD aligned */
789 test_cases[6] = MAKE_TEST(PMD, PTE, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS,
790 "4MB mremap - Source PMD-aligned, Destination PTE-aligned");
791 test_cases[7] = MAKE_TEST(PMD, _1MB, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS,
792 "4MB mremap - Source PMD-aligned, Destination 1MB-aligned");
793 test_cases[8] = MAKE_TEST(PMD, PMD, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS,
794 "4MB mremap - Source PMD-aligned, Destination PMD-aligned");
795
796 /* Src addr PUD aligned */
797 test_cases[9] = MAKE_TEST(PUD, PTE, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
798 "2GB mremap - Source PUD-aligned, Destination PTE-aligned");
799 test_cases[10] = MAKE_TEST(PUD, _1MB, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
800 "2GB mremap - Source PUD-aligned, Destination 1MB-aligned");
801 test_cases[11] = MAKE_TEST(PUD, PMD, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
802 "2GB mremap - Source PUD-aligned, Destination PMD-aligned");
803 test_cases[12] = MAKE_TEST(PUD, PUD, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
804 "2GB mremap - Source PUD-aligned, Destination PUD-aligned");
805
806 /* Src and Dest addr 1MB aligned. 5MB mremap. */
807 test_cases[13] = MAKE_TEST(_1MB, _1MB, _5MB, NON_OVERLAPPING, EXPECT_SUCCESS,
808 "5MB mremap - Source 1MB-aligned, Destination 1MB-aligned");
809
810 /* Src and Dest addr 1MB aligned. 5MB mremap. */
811 test_cases[14] = MAKE_TEST(_1MB, _1MB, _5MB, NON_OVERLAPPING, EXPECT_SUCCESS,
812 "5MB mremap - Source 1MB-aligned, Dest 1MB-aligned with 40MB Preamble");
813 test_cases[14].config.dest_preamble_size = 10 * _4MB;
814
815 perf_test_cases[0] = MAKE_TEST(page_size, page_size, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS,
816 "1GB mremap - Source PTE-aligned, Destination PTE-aligned");
817 /*
818 * mremap 1GB region - Page table level aligned time
819 * comparison.
820 */
821 perf_test_cases[1] = MAKE_TEST(PMD, PMD, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS,
822 "1GB mremap - Source PMD-aligned, Destination PMD-aligned");
823 perf_test_cases[2] = MAKE_TEST(PUD, PUD, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS,
824 "1GB mremap - Source PUD-aligned, Destination PUD-aligned");
825
826 run_perf_tests = (threshold_mb == VALIDATION_NO_THRESHOLD) ||
827 (threshold_mb * _1MB >= _1GB);
828
829 ksft_set_plan(ARRAY_SIZE(test_cases) + (run_perf_tests ?
830 ARRAY_SIZE(perf_test_cases) : 0) + num_expand_tests + num_misc_tests);
831
832 for (i = 0; i < ARRAY_SIZE(test_cases); i++)
833 run_mremap_test_case(test_cases[i], &failures, threshold_mb,
834 pattern_seed, rand_addr);
835
836 maps_fp = fopen("/proc/self/maps", "r");
837
838 if (maps_fp == NULL) {
839 munmap(rand_addr, rand_size);
840 ksft_exit_fail_msg("Failed to read /proc/self/maps: %s\n", strerror(errno));
841 }
842
843 mremap_expand_merge(maps_fp, page_size);
844 mremap_expand_merge_offset(maps_fp, page_size);
845
846 fclose(maps_fp);
847
848 mremap_move_within_range(pattern_seed, rand_addr);
849 mremap_move_1mb_from_start(pattern_seed, rand_addr);
850
851 if (run_perf_tests) {
852 ksft_print_msg("\n%s\n",
853 "mremap HAVE_MOVE_PMD/PUD optimization time comparison for 1GB region:");
854 for (i = 0; i < ARRAY_SIZE(perf_test_cases); i++)
855 run_mremap_test_case(perf_test_cases[i], &failures,
856 threshold_mb, pattern_seed,
857 rand_addr);
858 }
859
860 munmap(rand_addr, rand_size);
861
862 if (failures > 0)
863 ksft_exit_fail();
864 else
865 ksft_exit_pass();
866 }
867