1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <sys/mman.h> 4 #include <sys/prctl.h> 5 #include <sys/wait.h> 6 #include <stdbool.h> 7 #include <time.h> 8 #include <string.h> 9 #include <numa.h> 10 #include <unistd.h> 11 #include <fcntl.h> 12 #include <stdint.h> 13 #include <err.h> 14 15 #include "kselftest.h" 16 #include <include/vdso/time64.h> 17 #include "vm_util.h" 18 #include "hugepage_settings.h" 19 20 #define KSM_SYSFS_PATH "/sys/kernel/mm/ksm/" 21 #define KSM_FP(s) (KSM_SYSFS_PATH s) 22 #define KSM_SCAN_LIMIT_SEC_DEFAULT 120 23 #define KSM_PAGE_COUNT_DEFAULT 10l 24 #define KSM_PROT_STR_DEFAULT "rw" 25 #define KSM_USE_ZERO_PAGES_DEFAULT false 26 #define KSM_MERGE_ACROSS_NODES_DEFAULT true 27 #define KSM_MERGE_TYPE_DEFAULT 0 28 #define MB (1ul << 20) 29 30 struct ksm_sysfs { 31 unsigned long max_page_sharing; 32 unsigned long merge_across_nodes; 33 unsigned long pages_to_scan; 34 unsigned long run; 35 unsigned long sleep_millisecs; 36 unsigned long stable_node_chains_prune_millisecs; 37 unsigned long use_zero_pages; 38 }; 39 40 enum ksm_merge_type { 41 KSM_MERGE_MADVISE, 42 KSM_MERGE_PRCTL, 43 KSM_MERGE_LAST = KSM_MERGE_PRCTL 44 }; 45 46 enum ksm_test_name { 47 CHECK_KSM_MERGE, 48 CHECK_KSM_UNMERGE, 49 CHECK_KSM_GET_MERGE_TYPE, 50 CHECK_KSM_ZERO_PAGE_MERGE, 51 CHECK_KSM_NUMA_MERGE, 52 KSM_MERGE_TIME, 53 KSM_MERGE_TIME_HUGE_PAGES, 54 KSM_UNMERGE_TIME, 55 KSM_COW_TIME 56 }; 57 58 int debug; 59 60 static int ksm_write_sysfs(const char *file_path, unsigned long val) 61 { 62 return write_sysfs(file_path, val); 63 } 64 65 static int ksm_read_sysfs(const char *file_path, unsigned long *val) 66 { 67 return read_sysfs(file_path, val); 68 } 69 70 static void ksm_print_sysfs(void) 71 { 72 unsigned long max_page_sharing, pages_sharing, pages_shared; 73 unsigned long full_scans, pages_unshared, pages_volatile; 74 unsigned long stable_node_chains, stable_node_dups; 75 long general_profit; 76 77 if (ksm_read_sysfs(KSM_FP("pages_shared"), &pages_shared) || 78 ksm_read_sysfs(KSM_FP("pages_sharing"), &pages_sharing) || 79 ksm_read_sysfs(KSM_FP("max_page_sharing"), &max_page_sharing) || 80 ksm_read_sysfs(KSM_FP("full_scans"), &full_scans) || 81 ksm_read_sysfs(KSM_FP("pages_unshared"), &pages_unshared) || 82 ksm_read_sysfs(KSM_FP("pages_volatile"), &pages_volatile) || 83 ksm_read_sysfs(KSM_FP("stable_node_chains"), &stable_node_chains) || 84 ksm_read_sysfs(KSM_FP("stable_node_dups"), &stable_node_dups) || 85 ksm_read_sysfs(KSM_FP("general_profit"), (unsigned long *)&general_profit)) 86 return; 87 88 printf("pages_shared : %lu\n", pages_shared); 89 printf("pages_sharing : %lu\n", pages_sharing); 90 printf("max_page_sharing : %lu\n", max_page_sharing); 91 printf("full_scans : %lu\n", full_scans); 92 printf("pages_unshared : %lu\n", pages_unshared); 93 printf("pages_volatile : %lu\n", pages_volatile); 94 printf("stable_node_chains: %lu\n", stable_node_chains); 95 printf("stable_node_dups : %lu\n", stable_node_dups); 96 printf("general_profit : %ld\n", general_profit); 97 } 98 99 static void ksm_print_procfs(void) 100 { 101 const char *file_name = "/proc/self/ksm_stat"; 102 char buffer[512]; 103 FILE *f = fopen(file_name, "r"); 104 105 if (!f) { 106 fprintf(stderr, "f %s\n", file_name); 107 perror("fopen"); 108 return; 109 } 110 111 while (fgets(buffer, sizeof(buffer), f)) 112 printf("%s", buffer); 113 114 fclose(f); 115 } 116 117 static int str_to_prot(char *prot_str) 118 { 119 int prot = 0; 120 121 if ((strchr(prot_str, 'r')) != NULL) 122 prot |= PROT_READ; 123 if ((strchr(prot_str, 'w')) != NULL) 124 prot |= PROT_WRITE; 125 if ((strchr(prot_str, 'x')) != NULL) 126 prot |= PROT_EXEC; 127 128 return prot; 129 } 130 131 static void print_help(void) 132 { 133 printf("usage: ksm_tests [-h] <test type> [-a prot] [-p page_count] [-l timeout]\n" 134 "[-z use_zero_pages] [-m merge_across_nodes] [-s size]\n"); 135 136 printf("Supported <test type>:\n" 137 " -M (page merging)\n" 138 " -Z (zero pages merging)\n" 139 " -N (merging of pages in different NUMA nodes)\n" 140 " -U (page unmerging)\n" 141 " -P evaluate merging time and speed.\n" 142 " For this test, the size of duplicated memory area (in MiB)\n" 143 " must be provided using -s option\n" 144 " -H evaluate merging time and speed of area allocated mostly with huge pages\n" 145 " For this test, the size of duplicated memory area (in MiB)\n" 146 " must be provided using -s option\n" 147 " -D evaluate unmerging time and speed when disabling KSM.\n" 148 " For this test, the size of duplicated memory area (in MiB)\n" 149 " must be provided using -s option\n" 150 " -C evaluate the time required to break COW of merged pages.\n\n"); 151 152 printf(" -a: specify the access protections of pages.\n" 153 " <prot> must be of the form [rwx].\n" 154 " Default: %s\n", KSM_PROT_STR_DEFAULT); 155 printf(" -p: specify the number of pages to test.\n" 156 " Default: %ld\n", KSM_PAGE_COUNT_DEFAULT); 157 printf(" -l: limit the maximum running time (in seconds) for a test.\n" 158 " Default: %d seconds\n", KSM_SCAN_LIMIT_SEC_DEFAULT); 159 printf(" -z: change use_zero_pages tunable\n" 160 " Default: %d\n", KSM_USE_ZERO_PAGES_DEFAULT); 161 printf(" -m: change merge_across_nodes tunable\n" 162 " Default: %d\n", KSM_MERGE_ACROSS_NODES_DEFAULT); 163 printf(" -d: turn debugging output on\n"); 164 printf(" -s: the size of duplicated memory area (in MiB)\n"); 165 printf(" -t: KSM merge type\n" 166 " Default: 0\n" 167 " 0: madvise merging\n" 168 " 1: prctl merging\n"); 169 170 exit(0); 171 } 172 173 static void *allocate_memory(void *ptr, int prot, int mapping, char data, size_t map_size) 174 { 175 void *map_ptr = mmap(ptr, map_size, PROT_WRITE, mapping, -1, 0); 176 177 if (map_ptr == MAP_FAILED) { 178 ksft_perror("mmap"); 179 return NULL; 180 } 181 memset(map_ptr, data, map_size); 182 if (mprotect(map_ptr, map_size, prot)) { 183 ksft_perror("mprotect"); 184 munmap(map_ptr, map_size); 185 return NULL; 186 } 187 188 return map_ptr; 189 } 190 191 static int ksm_do_scan(int scan_count, struct timespec start_time, int timeout) 192 { 193 struct timespec cur_time; 194 unsigned long cur_scan, init_scan; 195 196 if (ksm_read_sysfs(KSM_FP("full_scans"), &init_scan)) 197 return 1; 198 cur_scan = init_scan; 199 200 while (cur_scan < init_scan + scan_count) { 201 if (ksm_read_sysfs(KSM_FP("full_scans"), &cur_scan)) 202 return 1; 203 if (clock_gettime(CLOCK_MONOTONIC_RAW, &cur_time)) { 204 ksft_perror("clock_gettime"); 205 return 1; 206 } 207 if ((cur_time.tv_sec - start_time.tv_sec) > timeout) { 208 ksft_print_msg("Scan time limit exceeded\n"); 209 return 1; 210 } 211 } 212 213 return 0; 214 } 215 216 static int ksm_merge_pages(int merge_type, void *addr, size_t size, 217 struct timespec start_time, int timeout) 218 { 219 if (merge_type == KSM_MERGE_MADVISE) { 220 if (madvise(addr, size, MADV_MERGEABLE)) { 221 ksft_perror("madvise"); 222 return 1; 223 } 224 } else if (merge_type == KSM_MERGE_PRCTL) { 225 if (prctl(PR_SET_MEMORY_MERGE, 1, 0, 0, 0)) { 226 ksft_perror("prctl"); 227 return 1; 228 } 229 } 230 231 if (ksm_write_sysfs(KSM_FP("run"), 1)) 232 return 1; 233 234 /* Since merging occurs only after 2 scans, make sure to get at least 2 full scans */ 235 if (ksm_do_scan(2, start_time, timeout)) 236 return 1; 237 238 return 0; 239 } 240 241 static int ksm_unmerge_pages(void *addr, size_t size, 242 struct timespec start_time, int timeout) 243 { 244 if (madvise(addr, size, MADV_UNMERGEABLE)) { 245 ksft_perror("madvise"); 246 return 1; 247 } 248 return 0; 249 } 250 251 static bool assert_ksm_pages_count(long dupl_page_count) 252 { 253 unsigned long max_page_sharing, pages_sharing, pages_shared; 254 255 if (ksm_read_sysfs(KSM_FP("pages_shared"), &pages_shared) || 256 ksm_read_sysfs(KSM_FP("pages_sharing"), &pages_sharing) || 257 ksm_read_sysfs(KSM_FP("max_page_sharing"), &max_page_sharing)) 258 return false; 259 260 if (debug) { 261 ksm_print_sysfs(); 262 ksm_print_procfs(); 263 } 264 265 /* 266 * Since there must be at least 2 pages for merging and 1 page can be 267 * shared with the limited number of pages (max_page_sharing), sometimes 268 * there are 'leftover' pages that cannot be merged. For example, if there 269 * are 11 pages and max_page_sharing = 10, then only 10 pages will be 270 * merged and the 11th page won't be affected. As a result, when the number 271 * of duplicate pages is divided by max_page_sharing and the remainder is 1, 272 * pages_shared and pages_sharing values will be equal between dupl_page_count 273 * and dupl_page_count - 1. 274 */ 275 if (dupl_page_count % max_page_sharing == 1 || dupl_page_count % max_page_sharing == 0) { 276 if (pages_shared == dupl_page_count / max_page_sharing && 277 pages_sharing == pages_shared * (max_page_sharing - 1)) 278 return true; 279 } else { 280 if (pages_shared == (dupl_page_count / max_page_sharing + 1) && 281 pages_sharing == dupl_page_count - pages_shared) 282 return true; 283 } 284 285 return false; 286 } 287 288 static int ksm_save_def(struct ksm_sysfs *ksm_sysfs) 289 { 290 if (ksm_read_sysfs(KSM_FP("max_page_sharing"), &ksm_sysfs->max_page_sharing) || 291 (numa_available() ? 0 : 292 ksm_read_sysfs(KSM_FP("merge_across_nodes"), &ksm_sysfs->merge_across_nodes)) || 293 ksm_read_sysfs(KSM_FP("sleep_millisecs"), &ksm_sysfs->sleep_millisecs) || 294 ksm_read_sysfs(KSM_FP("pages_to_scan"), &ksm_sysfs->pages_to_scan) || 295 ksm_read_sysfs(KSM_FP("run"), &ksm_sysfs->run) || 296 ksm_read_sysfs(KSM_FP("stable_node_chains_prune_millisecs"), 297 &ksm_sysfs->stable_node_chains_prune_millisecs) || 298 ksm_read_sysfs(KSM_FP("use_zero_pages"), &ksm_sysfs->use_zero_pages)) 299 return 1; 300 301 return 0; 302 } 303 304 static int ksm_restore(struct ksm_sysfs *ksm_sysfs) 305 { 306 if (ksm_write_sysfs(KSM_FP("max_page_sharing"), ksm_sysfs->max_page_sharing) || 307 (numa_available() ? 0 : 308 ksm_write_sysfs(KSM_FP("merge_across_nodes"), ksm_sysfs->merge_across_nodes)) || 309 ksm_write_sysfs(KSM_FP("pages_to_scan"), ksm_sysfs->pages_to_scan) || 310 ksm_write_sysfs(KSM_FP("run"), ksm_sysfs->run) || 311 ksm_write_sysfs(KSM_FP("sleep_millisecs"), ksm_sysfs->sleep_millisecs) || 312 ksm_write_sysfs(KSM_FP("stable_node_chains_prune_millisecs"), 313 ksm_sysfs->stable_node_chains_prune_millisecs) || 314 ksm_write_sysfs(KSM_FP("use_zero_pages"), ksm_sysfs->use_zero_pages)) 315 return 1; 316 317 return 0; 318 } 319 320 static int check_ksm_merge(int merge_type, int mapping, int prot, 321 long page_count, int timeout, size_t page_size) 322 { 323 void *map_ptr; 324 struct timespec start_time; 325 326 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 327 ksft_perror("clock_gettime"); 328 return KSFT_FAIL; 329 } 330 331 /* fill pages with the same data and merge them */ 332 map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count); 333 if (!map_ptr) 334 return KSFT_FAIL; 335 336 if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout)) 337 goto err_out; 338 339 /* verify that the right number of pages are merged */ 340 if (assert_ksm_pages_count(page_count)) { 341 munmap(map_ptr, page_size * page_count); 342 if (merge_type == KSM_MERGE_PRCTL) 343 prctl(PR_SET_MEMORY_MERGE, 0, 0, 0, 0); 344 return KSFT_PASS; 345 } 346 347 err_out: 348 munmap(map_ptr, page_size * page_count); 349 return KSFT_FAIL; 350 } 351 352 static int check_ksm_unmerge(int merge_type, int mapping, int prot, int timeout, size_t page_size) 353 { 354 void *map_ptr; 355 struct timespec start_time; 356 int page_count = 2; 357 358 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 359 ksft_perror("clock_gettime"); 360 return KSFT_FAIL; 361 } 362 363 /* fill pages with the same data and merge them */ 364 map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count); 365 if (!map_ptr) 366 return KSFT_FAIL; 367 368 if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout)) 369 goto err_out; 370 371 /* change 1 byte in each of the 2 pages -- KSM must automatically unmerge them */ 372 memset(map_ptr, '-', 1); 373 memset(map_ptr + page_size, '+', 1); 374 375 /* get at least 1 scan, so KSM can detect that the pages were modified */ 376 if (ksm_do_scan(1, start_time, timeout)) 377 goto err_out; 378 379 /* check that unmerging was successful and 0 pages are currently merged */ 380 if (assert_ksm_pages_count(0)) { 381 munmap(map_ptr, page_size * page_count); 382 return KSFT_PASS; 383 } 384 385 err_out: 386 munmap(map_ptr, page_size * page_count); 387 return KSFT_FAIL; 388 } 389 390 static int check_ksm_zero_page_merge(int merge_type, int mapping, int prot, long page_count, 391 int timeout, bool use_zero_pages, size_t page_size) 392 { 393 void *map_ptr; 394 struct timespec start_time; 395 396 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 397 ksft_perror("clock_gettime"); 398 return KSFT_FAIL; 399 } 400 401 if (ksm_write_sysfs(KSM_FP("use_zero_pages"), use_zero_pages)) 402 return KSFT_FAIL; 403 404 /* fill pages with zero and try to merge them */ 405 map_ptr = allocate_memory(NULL, prot, mapping, 0, page_size * page_count); 406 if (!map_ptr) 407 return KSFT_FAIL; 408 409 if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout)) 410 goto err_out; 411 412 /* 413 * verify that the right number of pages are merged: 414 * 1) if use_zero_pages is set to 1, empty pages are merged 415 * with the kernel zero page instead of with each other; 416 * 2) if use_zero_pages is set to 0, empty pages are not treated specially 417 * and merged as usual. 418 */ 419 if (use_zero_pages && !assert_ksm_pages_count(0)) 420 goto err_out; 421 else if (!use_zero_pages && !assert_ksm_pages_count(page_count)) 422 goto err_out; 423 424 munmap(map_ptr, page_size * page_count); 425 return KSFT_PASS; 426 427 err_out: 428 munmap(map_ptr, page_size * page_count); 429 return KSFT_FAIL; 430 } 431 432 static int get_next_mem_node(int node) 433 { 434 435 long node_size; 436 int mem_node = 0; 437 int i, max_node = numa_max_node(); 438 439 for (i = node + 1; i <= max_node + node; i++) { 440 mem_node = i % (max_node + 1); 441 node_size = numa_node_size(mem_node, NULL); 442 if (node_size > 0) 443 return mem_node; 444 } 445 return -ENODEV; 446 } 447 448 static int get_first_mem_node(void) 449 { 450 return get_next_mem_node(numa_max_node()); 451 } 452 453 static int check_ksm_numa_merge(int merge_type, int mapping, int prot, int timeout, 454 bool merge_across_nodes, size_t page_size) 455 { 456 void *numa1_map_ptr, *numa2_map_ptr; 457 struct timespec start_time; 458 int first_node, second_node; 459 int page_count = 2; 460 461 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 462 ksft_perror("clock_gettime"); 463 return KSFT_FAIL; 464 } 465 466 if (numa_available() < 0) { 467 ksft_print_msg("NUMA support not enabled\n"); 468 return KSFT_SKIP; 469 } 470 first_node = get_first_mem_node(); 471 second_node = get_next_mem_node(first_node); 472 473 if (second_node < 0) { 474 ksft_print_msg("At least 2 NUMA nodes with memory must be available\n"); 475 return KSFT_SKIP; 476 } 477 if (ksm_write_sysfs(KSM_FP("merge_across_nodes"), merge_across_nodes)) 478 return KSFT_FAIL; 479 480 /* allocate 2 pages in 2 different NUMA nodes and fill them with the same data */ 481 numa1_map_ptr = numa_alloc_onnode(page_size, first_node); 482 numa2_map_ptr = numa_alloc_onnode(page_size, second_node); 483 if (!numa1_map_ptr || !numa2_map_ptr) { 484 ksft_perror("numa_alloc_onnode"); 485 return KSFT_FAIL; 486 } 487 488 memset(numa1_map_ptr, '*', page_size); 489 memset(numa2_map_ptr, '*', page_size); 490 491 /* try to merge the pages */ 492 if (ksm_merge_pages(merge_type, numa1_map_ptr, page_size, start_time, timeout) || 493 ksm_merge_pages(merge_type, numa2_map_ptr, page_size, start_time, timeout)) 494 goto err_out; 495 496 /* 497 * verify that the right number of pages are merged: 498 * 1) if merge_across_nodes was enabled, 2 duplicate pages will be merged; 499 * 2) if merge_across_nodes = 0, there must be 0 merged pages, since there is 500 * only 1 unique page in each node and they can't be shared. 501 */ 502 if (merge_across_nodes && !assert_ksm_pages_count(page_count)) 503 goto err_out; 504 else if (!merge_across_nodes && !assert_ksm_pages_count(0)) 505 goto err_out; 506 507 numa_free(numa1_map_ptr, page_size); 508 numa_free(numa2_map_ptr, page_size); 509 return KSFT_PASS; 510 511 err_out: 512 numa_free(numa1_map_ptr, page_size); 513 numa_free(numa2_map_ptr, page_size); 514 return KSFT_FAIL; 515 } 516 517 static int ksm_merge_hugepages_time(int merge_type, int mapping, int prot, 518 int timeout, size_t map_size) 519 { 520 void *map_ptr, *map_ptr_orig; 521 struct timespec start_time, end_time; 522 unsigned long scan_time_ns; 523 int pagemap_fd, n_normal_pages, n_huge_pages; 524 525 if (!thp_is_enabled()) { 526 ksft_print_msg("Transparent Hugepages not available\n"); 527 return KSFT_SKIP; 528 } 529 530 map_size *= MB; 531 size_t len = map_size; 532 533 len -= len % HPAGE_SIZE; 534 map_ptr_orig = mmap(NULL, len + HPAGE_SIZE, PROT_READ | PROT_WRITE, 535 MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE, -1, 0); 536 map_ptr = map_ptr_orig + HPAGE_SIZE - (uintptr_t)map_ptr_orig % HPAGE_SIZE; 537 538 if (map_ptr_orig == MAP_FAILED) 539 err(2, "initial mmap"); 540 541 if (madvise(map_ptr, len, MADV_HUGEPAGE)) 542 err(2, "MADV_HUGEPAGE"); 543 544 pagemap_fd = open("/proc/self/pagemap", O_RDONLY); 545 if (pagemap_fd < 0) 546 err(2, "open pagemap"); 547 548 n_normal_pages = 0; 549 n_huge_pages = 0; 550 for (void *p = map_ptr; p < map_ptr + len; p += HPAGE_SIZE) { 551 if (allocate_transhuge(p, pagemap_fd) < 0) 552 n_normal_pages++; 553 else 554 n_huge_pages++; 555 } 556 ksft_print_msg("Number of normal pages: %d\n", n_normal_pages); 557 ksft_print_msg("Number of huge pages: %d\n", n_huge_pages); 558 559 memset(map_ptr, '*', len); 560 561 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 562 ksft_perror("clock_gettime"); 563 goto err_out; 564 } 565 if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout)) 566 goto err_out; 567 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { 568 ksft_perror("clock_gettime"); 569 goto err_out; 570 } 571 572 scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + 573 (end_time.tv_nsec - start_time.tv_nsec); 574 575 ksft_print_msg("Total size: %lu MiB\n", map_size / MB); 576 ksft_print_msg("Total time: %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC, 577 scan_time_ns % NSEC_PER_SEC); 578 ksft_print_msg("Average speed: %.3f MiB/s\n", (map_size / MB) / 579 ((double)scan_time_ns / NSEC_PER_SEC)); 580 581 munmap(map_ptr_orig, len + HPAGE_SIZE); 582 return KSFT_PASS; 583 584 err_out: 585 munmap(map_ptr_orig, len + HPAGE_SIZE); 586 return KSFT_FAIL; 587 } 588 589 static int ksm_merge_time(int merge_type, int mapping, int prot, int timeout, size_t map_size) 590 { 591 void *map_ptr; 592 struct timespec start_time, end_time; 593 unsigned long scan_time_ns; 594 595 map_size *= MB; 596 597 map_ptr = allocate_memory(NULL, prot, mapping, '*', map_size); 598 if (!map_ptr) 599 return KSFT_FAIL; 600 601 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 602 ksft_perror("clock_gettime"); 603 goto err_out; 604 } 605 if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout)) 606 goto err_out; 607 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { 608 ksft_perror("clock_gettime"); 609 goto err_out; 610 } 611 612 scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + 613 (end_time.tv_nsec - start_time.tv_nsec); 614 615 ksft_print_msg("Total size: %lu MiB\n", map_size / MB); 616 ksft_print_msg("Total time: %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC, 617 scan_time_ns % NSEC_PER_SEC); 618 ksft_print_msg("Average speed: %.3f MiB/s\n", (map_size / MB) / 619 ((double)scan_time_ns / NSEC_PER_SEC)); 620 621 munmap(map_ptr, map_size); 622 return KSFT_PASS; 623 624 err_out: 625 munmap(map_ptr, map_size); 626 return KSFT_FAIL; 627 } 628 629 static int ksm_unmerge_time(int merge_type, int mapping, int prot, int timeout, size_t map_size) 630 { 631 void *map_ptr; 632 struct timespec start_time, end_time; 633 unsigned long scan_time_ns; 634 635 map_size *= MB; 636 637 map_ptr = allocate_memory(NULL, prot, mapping, '*', map_size); 638 if (!map_ptr) 639 return KSFT_FAIL; 640 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 641 ksft_perror("clock_gettime"); 642 goto err_out; 643 } 644 if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout)) 645 goto err_out; 646 647 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 648 ksft_perror("clock_gettime"); 649 goto err_out; 650 } 651 if (ksm_unmerge_pages(map_ptr, map_size, start_time, timeout)) 652 goto err_out; 653 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { 654 ksft_perror("clock_gettime"); 655 goto err_out; 656 } 657 658 scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + 659 (end_time.tv_nsec - start_time.tv_nsec); 660 661 ksft_print_msg("Total size: %lu MiB\n", map_size / MB); 662 ksft_print_msg("Total time: %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC, 663 scan_time_ns % NSEC_PER_SEC); 664 ksft_print_msg("Average speed: %.3f MiB/s\n", (map_size / MB) / 665 ((double)scan_time_ns / NSEC_PER_SEC)); 666 667 munmap(map_ptr, map_size); 668 return KSFT_PASS; 669 670 err_out: 671 munmap(map_ptr, map_size); 672 return KSFT_FAIL; 673 } 674 675 static int ksm_cow_time(int merge_type, int mapping, int prot, int timeout, size_t page_size) 676 { 677 void *map_ptr; 678 struct timespec start_time, end_time; 679 unsigned long cow_time_ns; 680 681 /* page_count must be less than 2*page_size */ 682 size_t page_count = 4000; 683 684 map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count); 685 if (!map_ptr) 686 return KSFT_FAIL; 687 688 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 689 ksft_perror("clock_gettime"); 690 return KSFT_FAIL; 691 } 692 for (size_t i = 0; i < page_count - 1; i = i + 2) 693 memset(map_ptr + page_size * i, '-', 1); 694 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { 695 ksft_perror("clock_gettime"); 696 return KSFT_FAIL; 697 } 698 699 cow_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + 700 (end_time.tv_nsec - start_time.tv_nsec); 701 702 ksft_print_msg("Total size: %lu MiB\n\n", (page_size * page_count) / MB); 703 ksft_print_msg("Not merged pages:\n"); 704 ksft_print_msg("Total time: %ld.%09ld s\n", cow_time_ns / NSEC_PER_SEC, 705 cow_time_ns % NSEC_PER_SEC); 706 ksft_print_msg("Average speed: %.3f MiB/s\n\n", ((page_size * (page_count / 2)) / MB) / 707 ((double)cow_time_ns / NSEC_PER_SEC)); 708 709 /* Create 2000 pairs of duplicate pages */ 710 for (size_t i = 0; i < page_count - 1; i = i + 2) { 711 memset(map_ptr + page_size * i, '+', i / 2 + 1); 712 memset(map_ptr + page_size * (i + 1), '+', i / 2 + 1); 713 } 714 if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout)) 715 goto err_out; 716 717 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 718 ksft_perror("clock_gettime"); 719 goto err_out; 720 } 721 for (size_t i = 0; i < page_count - 1; i = i + 2) 722 memset(map_ptr + page_size * i, '-', 1); 723 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { 724 ksft_perror("clock_gettime"); 725 goto err_out; 726 } 727 728 cow_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + 729 (end_time.tv_nsec - start_time.tv_nsec); 730 731 ksft_print_msg("Merged pages:\n"); 732 ksft_print_msg("Total time: %ld.%09ld s\n", cow_time_ns / NSEC_PER_SEC, 733 cow_time_ns % NSEC_PER_SEC); 734 ksft_print_msg("Average speed: %.3f MiB/s\n", ((page_size * (page_count / 2)) / MB) / 735 ((double)cow_time_ns / NSEC_PER_SEC)); 736 737 munmap(map_ptr, page_size * page_count); 738 return KSFT_PASS; 739 740 err_out: 741 munmap(map_ptr, page_size * page_count); 742 return KSFT_FAIL; 743 } 744 745 int main(int argc, char *argv[]) 746 { 747 int ret = 0, opt; 748 int prot = 0; 749 int ksm_scan_limit_sec = KSM_SCAN_LIMIT_SEC_DEFAULT; 750 int merge_type = KSM_MERGE_TYPE_DEFAULT; 751 long page_count = KSM_PAGE_COUNT_DEFAULT; 752 size_t page_size = sysconf(_SC_PAGESIZE); 753 struct ksm_sysfs ksm_sysfs_old; 754 int test_name = CHECK_KSM_MERGE; 755 bool use_zero_pages = KSM_USE_ZERO_PAGES_DEFAULT; 756 bool merge_across_nodes = KSM_MERGE_ACROSS_NODES_DEFAULT; 757 long size_MB = 0; 758 const char *test_descr = "KSM merging"; 759 760 ksft_print_header(); 761 ksft_set_plan(1); 762 763 while ((opt = getopt(argc, argv, "dha:p:l:z:m:s:t:MUZNPCHD")) != -1) { 764 switch (opt) { 765 case 'a': 766 prot = str_to_prot(optarg); 767 break; 768 case 'p': 769 page_count = atol(optarg); 770 if (page_count <= 0) 771 ksft_exit_fail_msg("The number of pages must be greater than 0\n"); 772 break; 773 case 'l': 774 ksm_scan_limit_sec = atoi(optarg); 775 if (ksm_scan_limit_sec <= 0) 776 ksft_exit_fail_msg("Timeout value must be greater than 0\n"); 777 break; 778 case 'h': 779 print_help(); 780 break; 781 case 'z': 782 if (strcmp(optarg, "0") == 0) 783 use_zero_pages = 0; 784 else 785 use_zero_pages = 1; 786 break; 787 case 'm': 788 if (strcmp(optarg, "0") == 0) 789 merge_across_nodes = 0; 790 else 791 merge_across_nodes = 1; 792 break; 793 case 'd': 794 debug = 1; 795 break; 796 case 's': 797 size_MB = atoi(optarg); 798 if (size_MB <= 0) 799 ksft_exit_fail_msg("Size must be greater than 0\n"); 800 break; 801 case 't': 802 { 803 int tmp = atoi(optarg); 804 805 if (tmp < 0 || tmp > KSM_MERGE_LAST) 806 ksft_exit_fail_msg("Invalid merge type\n"); 807 merge_type = tmp; 808 } 809 break; 810 case 'M': 811 break; 812 case 'U': 813 test_name = CHECK_KSM_UNMERGE; 814 break; 815 case 'Z': 816 test_name = CHECK_KSM_ZERO_PAGE_MERGE; 817 break; 818 case 'N': 819 test_name = CHECK_KSM_NUMA_MERGE; 820 break; 821 case 'P': 822 test_name = KSM_MERGE_TIME; 823 break; 824 case 'H': 825 test_name = KSM_MERGE_TIME_HUGE_PAGES; 826 break; 827 case 'D': 828 test_name = KSM_UNMERGE_TIME; 829 break; 830 case 'C': 831 test_name = KSM_COW_TIME; 832 break; 833 default: 834 ksft_exit_fail_msg("Unknown option\n"); 835 } 836 } 837 838 if (prot == 0) 839 prot = str_to_prot(KSM_PROT_STR_DEFAULT); 840 841 if (access(KSM_SYSFS_PATH, F_OK)) 842 ksft_exit_skip("Config KSM not enabled\n"); 843 844 if (ksm_save_def(&ksm_sysfs_old)) 845 ksft_exit_fail_msg("Cannot save default tunables\n"); 846 847 if (ksm_write_sysfs(KSM_FP("run"), 2) || 848 ksm_write_sysfs(KSM_FP("sleep_millisecs"), 0) || 849 (numa_available() ? 0 : 850 ksm_write_sysfs(KSM_FP("merge_across_nodes"), 1)) || 851 ksm_write_sysfs(KSM_FP("pages_to_scan"), page_count)) 852 ksft_exit_fail_msg("Cannot set up KSM tunables\n"); 853 854 switch (test_name) { 855 case CHECK_KSM_MERGE: 856 test_descr = "KSM merging"; 857 ret = check_ksm_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, page_count, 858 ksm_scan_limit_sec, page_size); 859 break; 860 case CHECK_KSM_UNMERGE: 861 test_descr = "KSM unmerging"; 862 ret = check_ksm_unmerge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, 863 ksm_scan_limit_sec, page_size); 864 break; 865 case CHECK_KSM_ZERO_PAGE_MERGE: 866 test_descr = "KSM zero page merging"; 867 ret = check_ksm_zero_page_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, 868 page_count, ksm_scan_limit_sec, use_zero_pages, 869 page_size); 870 break; 871 case CHECK_KSM_NUMA_MERGE: 872 test_descr = "KSM NUMA merging"; 873 ret = check_ksm_numa_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, 874 ksm_scan_limit_sec, merge_across_nodes, page_size); 875 break; 876 case KSM_MERGE_TIME: 877 if (size_MB == 0) 878 ksft_exit_fail_msg("Option '-s' is required\n"); 879 test_descr = "KSM merge time"; 880 ret = ksm_merge_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, 881 ksm_scan_limit_sec, size_MB); 882 break; 883 case KSM_MERGE_TIME_HUGE_PAGES: 884 if (size_MB == 0) 885 ksft_exit_fail_msg("Option '-s' is required\n"); 886 test_descr = "KSM merge time with huge pages"; 887 ret = ksm_merge_hugepages_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, 888 ksm_scan_limit_sec, size_MB); 889 break; 890 case KSM_UNMERGE_TIME: 891 if (size_MB == 0) 892 ksft_exit_fail_msg("Option '-s' is required\n"); 893 test_descr = "KSM unmerge time"; 894 ret = ksm_unmerge_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, 895 ksm_scan_limit_sec, size_MB); 896 break; 897 case KSM_COW_TIME: 898 test_descr = "KSM COW time"; 899 ret = ksm_cow_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, 900 ksm_scan_limit_sec, page_size); 901 break; 902 } 903 904 if (ksm_restore(&ksm_sysfs_old)) 905 ksft_print_msg("Cannot restore default tunables\n"); 906 907 ksft_test_result_report(ret, "%s\n", test_descr); 908 909 ksft_finished(); 910 } 911