1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <sys/mman.h> 4 #include <sys/prctl.h> 5 #include <sys/wait.h> 6 #include <stdbool.h> 7 #include <time.h> 8 #include <string.h> 9 #include <numa.h> 10 #include <unistd.h> 11 #include <fcntl.h> 12 #include <stdint.h> 13 #include <err.h> 14 15 #include "../kselftest.h" 16 #include <include/vdso/time64.h> 17 #include "vm_util.h" 18 #include "thp_settings.h" 19 20 #define KSM_SYSFS_PATH "/sys/kernel/mm/ksm/" 21 #define KSM_FP(s) (KSM_SYSFS_PATH s) 22 #define KSM_SCAN_LIMIT_SEC_DEFAULT 120 23 #define KSM_PAGE_COUNT_DEFAULT 10l 24 #define KSM_PROT_STR_DEFAULT "rw" 25 #define KSM_USE_ZERO_PAGES_DEFAULT false 26 #define KSM_MERGE_ACROSS_NODES_DEFAULT true 27 #define KSM_MERGE_TYPE_DEFAULT 0 28 #define MB (1ul << 20) 29 30 struct ksm_sysfs { 31 unsigned long max_page_sharing; 32 unsigned long merge_across_nodes; 33 unsigned long pages_to_scan; 34 unsigned long run; 35 unsigned long sleep_millisecs; 36 unsigned long stable_node_chains_prune_millisecs; 37 unsigned long use_zero_pages; 38 }; 39 40 enum ksm_merge_type { 41 KSM_MERGE_MADVISE, 42 KSM_MERGE_PRCTL, 43 KSM_MERGE_LAST = KSM_MERGE_PRCTL 44 }; 45 46 enum ksm_test_name { 47 CHECK_KSM_MERGE, 48 CHECK_KSM_UNMERGE, 49 CHECK_KSM_GET_MERGE_TYPE, 50 CHECK_KSM_ZERO_PAGE_MERGE, 51 CHECK_KSM_NUMA_MERGE, 52 KSM_MERGE_TIME, 53 KSM_MERGE_TIME_HUGE_PAGES, 54 KSM_UNMERGE_TIME, 55 KSM_COW_TIME 56 }; 57 58 int debug; 59 60 static int ksm_write_sysfs(const char *file_path, unsigned long val) 61 { 62 return write_sysfs(file_path, val); 63 } 64 65 static int ksm_read_sysfs(const char *file_path, unsigned long *val) 66 { 67 return read_sysfs(file_path, val); 68 } 69 70 static void ksm_print_sysfs(void) 71 { 72 unsigned long max_page_sharing, pages_sharing, pages_shared; 73 unsigned long full_scans, pages_unshared, pages_volatile; 74 unsigned long stable_node_chains, stable_node_dups; 75 long general_profit; 76 77 if (ksm_read_sysfs(KSM_FP("pages_shared"), &pages_shared) || 78 ksm_read_sysfs(KSM_FP("pages_sharing"), &pages_sharing) || 79 ksm_read_sysfs(KSM_FP("max_page_sharing"), &max_page_sharing) || 80 ksm_read_sysfs(KSM_FP("full_scans"), &full_scans) || 81 ksm_read_sysfs(KSM_FP("pages_unshared"), &pages_unshared) || 82 ksm_read_sysfs(KSM_FP("pages_volatile"), &pages_volatile) || 83 ksm_read_sysfs(KSM_FP("stable_node_chains"), &stable_node_chains) || 84 ksm_read_sysfs(KSM_FP("stable_node_dups"), &stable_node_dups) || 85 ksm_read_sysfs(KSM_FP("general_profit"), (unsigned long *)&general_profit)) 86 return; 87 88 printf("pages_shared : %lu\n", pages_shared); 89 printf("pages_sharing : %lu\n", pages_sharing); 90 printf("max_page_sharing : %lu\n", max_page_sharing); 91 printf("full_scans : %lu\n", full_scans); 92 printf("pages_unshared : %lu\n", pages_unshared); 93 printf("pages_volatile : %lu\n", pages_volatile); 94 printf("stable_node_chains: %lu\n", stable_node_chains); 95 printf("stable_node_dups : %lu\n", stable_node_dups); 96 printf("general_profit : %ld\n", general_profit); 97 } 98 99 static void ksm_print_procfs(void) 100 { 101 const char *file_name = "/proc/self/ksm_stat"; 102 char buffer[512]; 103 FILE *f = fopen(file_name, "r"); 104 105 if (!f) { 106 fprintf(stderr, "f %s\n", file_name); 107 perror("fopen"); 108 return; 109 } 110 111 while (fgets(buffer, sizeof(buffer), f)) 112 printf("%s", buffer); 113 114 fclose(f); 115 } 116 117 static int str_to_prot(char *prot_str) 118 { 119 int prot = 0; 120 121 if ((strchr(prot_str, 'r')) != NULL) 122 prot |= PROT_READ; 123 if ((strchr(prot_str, 'w')) != NULL) 124 prot |= PROT_WRITE; 125 if ((strchr(prot_str, 'x')) != NULL) 126 prot |= PROT_EXEC; 127 128 return prot; 129 } 130 131 static void print_help(void) 132 { 133 printf("usage: ksm_tests [-h] <test type> [-a prot] [-p page_count] [-l timeout]\n" 134 "[-z use_zero_pages] [-m merge_across_nodes] [-s size]\n"); 135 136 printf("Supported <test type>:\n" 137 " -M (page merging)\n" 138 " -Z (zero pages merging)\n" 139 " -N (merging of pages in different NUMA nodes)\n" 140 " -U (page unmerging)\n" 141 " -P evaluate merging time and speed.\n" 142 " For this test, the size of duplicated memory area (in MiB)\n" 143 " must be provided using -s option\n" 144 " -H evaluate merging time and speed of area allocated mostly with huge pages\n" 145 " For this test, the size of duplicated memory area (in MiB)\n" 146 " must be provided using -s option\n" 147 " -D evaluate unmerging time and speed when disabling KSM.\n" 148 " For this test, the size of duplicated memory area (in MiB)\n" 149 " must be provided using -s option\n" 150 " -C evaluate the time required to break COW of merged pages.\n\n"); 151 152 printf(" -a: specify the access protections of pages.\n" 153 " <prot> must be of the form [rwx].\n" 154 " Default: %s\n", KSM_PROT_STR_DEFAULT); 155 printf(" -p: specify the number of pages to test.\n" 156 " Default: %ld\n", KSM_PAGE_COUNT_DEFAULT); 157 printf(" -l: limit the maximum running time (in seconds) for a test.\n" 158 " Default: %d seconds\n", KSM_SCAN_LIMIT_SEC_DEFAULT); 159 printf(" -z: change use_zero_pages tunable\n" 160 " Default: %d\n", KSM_USE_ZERO_PAGES_DEFAULT); 161 printf(" -m: change merge_across_nodes tunable\n" 162 " Default: %d\n", KSM_MERGE_ACROSS_NODES_DEFAULT); 163 printf(" -d: turn debugging output on\n"); 164 printf(" -s: the size of duplicated memory area (in MiB)\n"); 165 printf(" -t: KSM merge type\n" 166 " Default: 0\n" 167 " 0: madvise merging\n" 168 " 1: prctl merging\n"); 169 170 exit(0); 171 } 172 173 static void *allocate_memory(void *ptr, int prot, int mapping, char data, size_t map_size) 174 { 175 void *map_ptr = mmap(ptr, map_size, PROT_WRITE, mapping, -1, 0); 176 177 if (!map_ptr) { 178 perror("mmap"); 179 return NULL; 180 } 181 memset(map_ptr, data, map_size); 182 if (mprotect(map_ptr, map_size, prot)) { 183 perror("mprotect"); 184 munmap(map_ptr, map_size); 185 return NULL; 186 } 187 188 return map_ptr; 189 } 190 191 static int ksm_do_scan(int scan_count, struct timespec start_time, int timeout) 192 { 193 struct timespec cur_time; 194 unsigned long cur_scan, init_scan; 195 196 if (ksm_read_sysfs(KSM_FP("full_scans"), &init_scan)) 197 return 1; 198 cur_scan = init_scan; 199 200 while (cur_scan < init_scan + scan_count) { 201 if (ksm_read_sysfs(KSM_FP("full_scans"), &cur_scan)) 202 return 1; 203 if (clock_gettime(CLOCK_MONOTONIC_RAW, &cur_time)) { 204 perror("clock_gettime"); 205 return 1; 206 } 207 if ((cur_time.tv_sec - start_time.tv_sec) > timeout) { 208 printf("Scan time limit exceeded\n"); 209 return 1; 210 } 211 } 212 213 return 0; 214 } 215 216 static int ksm_merge_pages(int merge_type, void *addr, size_t size, 217 struct timespec start_time, int timeout) 218 { 219 if (merge_type == KSM_MERGE_MADVISE) { 220 if (madvise(addr, size, MADV_MERGEABLE)) { 221 perror("madvise"); 222 return 1; 223 } 224 } else if (merge_type == KSM_MERGE_PRCTL) { 225 if (prctl(PR_SET_MEMORY_MERGE, 1, 0, 0, 0)) { 226 perror("prctl"); 227 return 1; 228 } 229 } 230 231 if (ksm_write_sysfs(KSM_FP("run"), 1)) 232 return 1; 233 234 /* Since merging occurs only after 2 scans, make sure to get at least 2 full scans */ 235 if (ksm_do_scan(2, start_time, timeout)) 236 return 1; 237 238 return 0; 239 } 240 241 static int ksm_unmerge_pages(void *addr, size_t size, 242 struct timespec start_time, int timeout) 243 { 244 if (madvise(addr, size, MADV_UNMERGEABLE)) { 245 perror("madvise"); 246 return 1; 247 } 248 return 0; 249 } 250 251 static bool assert_ksm_pages_count(long dupl_page_count) 252 { 253 unsigned long max_page_sharing, pages_sharing, pages_shared; 254 255 if (ksm_read_sysfs(KSM_FP("pages_shared"), &pages_shared) || 256 ksm_read_sysfs(KSM_FP("pages_sharing"), &pages_sharing) || 257 ksm_read_sysfs(KSM_FP("max_page_sharing"), &max_page_sharing)) 258 return false; 259 260 if (debug) { 261 ksm_print_sysfs(); 262 ksm_print_procfs(); 263 } 264 265 /* 266 * Since there must be at least 2 pages for merging and 1 page can be 267 * shared with the limited number of pages (max_page_sharing), sometimes 268 * there are 'leftover' pages that cannot be merged. For example, if there 269 * are 11 pages and max_page_sharing = 10, then only 10 pages will be 270 * merged and the 11th page won't be affected. As a result, when the number 271 * of duplicate pages is divided by max_page_sharing and the remainder is 1, 272 * pages_shared and pages_sharing values will be equal between dupl_page_count 273 * and dupl_page_count - 1. 274 */ 275 if (dupl_page_count % max_page_sharing == 1 || dupl_page_count % max_page_sharing == 0) { 276 if (pages_shared == dupl_page_count / max_page_sharing && 277 pages_sharing == pages_shared * (max_page_sharing - 1)) 278 return true; 279 } else { 280 if (pages_shared == (dupl_page_count / max_page_sharing + 1) && 281 pages_sharing == dupl_page_count - pages_shared) 282 return true; 283 } 284 285 return false; 286 } 287 288 static int ksm_save_def(struct ksm_sysfs *ksm_sysfs) 289 { 290 if (ksm_read_sysfs(KSM_FP("max_page_sharing"), &ksm_sysfs->max_page_sharing) || 291 numa_available() ? 0 : 292 ksm_read_sysfs(KSM_FP("merge_across_nodes"), &ksm_sysfs->merge_across_nodes) || 293 ksm_read_sysfs(KSM_FP("sleep_millisecs"), &ksm_sysfs->sleep_millisecs) || 294 ksm_read_sysfs(KSM_FP("pages_to_scan"), &ksm_sysfs->pages_to_scan) || 295 ksm_read_sysfs(KSM_FP("run"), &ksm_sysfs->run) || 296 ksm_read_sysfs(KSM_FP("stable_node_chains_prune_millisecs"), 297 &ksm_sysfs->stable_node_chains_prune_millisecs) || 298 ksm_read_sysfs(KSM_FP("use_zero_pages"), &ksm_sysfs->use_zero_pages)) 299 return 1; 300 301 return 0; 302 } 303 304 static int ksm_restore(struct ksm_sysfs *ksm_sysfs) 305 { 306 if (ksm_write_sysfs(KSM_FP("max_page_sharing"), ksm_sysfs->max_page_sharing) || 307 numa_available() ? 0 : 308 ksm_write_sysfs(KSM_FP("merge_across_nodes"), ksm_sysfs->merge_across_nodes) || 309 ksm_write_sysfs(KSM_FP("pages_to_scan"), ksm_sysfs->pages_to_scan) || 310 ksm_write_sysfs(KSM_FP("run"), ksm_sysfs->run) || 311 ksm_write_sysfs(KSM_FP("sleep_millisecs"), ksm_sysfs->sleep_millisecs) || 312 ksm_write_sysfs(KSM_FP("stable_node_chains_prune_millisecs"), 313 ksm_sysfs->stable_node_chains_prune_millisecs) || 314 ksm_write_sysfs(KSM_FP("use_zero_pages"), ksm_sysfs->use_zero_pages)) 315 return 1; 316 317 return 0; 318 } 319 320 static int check_ksm_merge(int merge_type, int mapping, int prot, 321 long page_count, int timeout, size_t page_size) 322 { 323 void *map_ptr; 324 struct timespec start_time; 325 326 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 327 perror("clock_gettime"); 328 return KSFT_FAIL; 329 } 330 331 /* fill pages with the same data and merge them */ 332 map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count); 333 if (!map_ptr) 334 return KSFT_FAIL; 335 336 if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout)) 337 goto err_out; 338 339 /* verify that the right number of pages are merged */ 340 if (assert_ksm_pages_count(page_count)) { 341 printf("OK\n"); 342 munmap(map_ptr, page_size * page_count); 343 if (merge_type == KSM_MERGE_PRCTL) 344 prctl(PR_SET_MEMORY_MERGE, 0, 0, 0, 0); 345 return KSFT_PASS; 346 } 347 348 err_out: 349 printf("Not OK\n"); 350 munmap(map_ptr, page_size * page_count); 351 return KSFT_FAIL; 352 } 353 354 static int check_ksm_unmerge(int merge_type, int mapping, int prot, int timeout, size_t page_size) 355 { 356 void *map_ptr; 357 struct timespec start_time; 358 int page_count = 2; 359 360 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 361 perror("clock_gettime"); 362 return KSFT_FAIL; 363 } 364 365 /* fill pages with the same data and merge them */ 366 map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count); 367 if (!map_ptr) 368 return KSFT_FAIL; 369 370 if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout)) 371 goto err_out; 372 373 /* change 1 byte in each of the 2 pages -- KSM must automatically unmerge them */ 374 memset(map_ptr, '-', 1); 375 memset(map_ptr + page_size, '+', 1); 376 377 /* get at least 1 scan, so KSM can detect that the pages were modified */ 378 if (ksm_do_scan(1, start_time, timeout)) 379 goto err_out; 380 381 /* check that unmerging was successful and 0 pages are currently merged */ 382 if (assert_ksm_pages_count(0)) { 383 printf("OK\n"); 384 munmap(map_ptr, page_size * page_count); 385 return KSFT_PASS; 386 } 387 388 err_out: 389 printf("Not OK\n"); 390 munmap(map_ptr, page_size * page_count); 391 return KSFT_FAIL; 392 } 393 394 static int check_ksm_zero_page_merge(int merge_type, int mapping, int prot, long page_count, 395 int timeout, bool use_zero_pages, size_t page_size) 396 { 397 void *map_ptr; 398 struct timespec start_time; 399 400 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 401 perror("clock_gettime"); 402 return KSFT_FAIL; 403 } 404 405 if (ksm_write_sysfs(KSM_FP("use_zero_pages"), use_zero_pages)) 406 return KSFT_FAIL; 407 408 /* fill pages with zero and try to merge them */ 409 map_ptr = allocate_memory(NULL, prot, mapping, 0, page_size * page_count); 410 if (!map_ptr) 411 return KSFT_FAIL; 412 413 if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout)) 414 goto err_out; 415 416 /* 417 * verify that the right number of pages are merged: 418 * 1) if use_zero_pages is set to 1, empty pages are merged 419 * with the kernel zero page instead of with each other; 420 * 2) if use_zero_pages is set to 0, empty pages are not treated specially 421 * and merged as usual. 422 */ 423 if (use_zero_pages && !assert_ksm_pages_count(0)) 424 goto err_out; 425 else if (!use_zero_pages && !assert_ksm_pages_count(page_count)) 426 goto err_out; 427 428 printf("OK\n"); 429 munmap(map_ptr, page_size * page_count); 430 return KSFT_PASS; 431 432 err_out: 433 printf("Not OK\n"); 434 munmap(map_ptr, page_size * page_count); 435 return KSFT_FAIL; 436 } 437 438 static int get_next_mem_node(int node) 439 { 440 441 long node_size; 442 int mem_node = 0; 443 int i, max_node = numa_max_node(); 444 445 for (i = node + 1; i <= max_node + node; i++) { 446 mem_node = i % (max_node + 1); 447 node_size = numa_node_size(mem_node, NULL); 448 if (node_size > 0) 449 break; 450 } 451 return mem_node; 452 } 453 454 static int get_first_mem_node(void) 455 { 456 return get_next_mem_node(numa_max_node()); 457 } 458 459 static int check_ksm_numa_merge(int merge_type, int mapping, int prot, int timeout, 460 bool merge_across_nodes, size_t page_size) 461 { 462 void *numa1_map_ptr, *numa2_map_ptr; 463 struct timespec start_time; 464 int page_count = 2; 465 int first_node; 466 467 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 468 perror("clock_gettime"); 469 return KSFT_FAIL; 470 } 471 472 if (numa_available() < 0) { 473 perror("NUMA support not enabled"); 474 return KSFT_SKIP; 475 } 476 if (numa_num_configured_nodes() <= 1) { 477 printf("At least 2 NUMA nodes must be available\n"); 478 return KSFT_SKIP; 479 } 480 if (ksm_write_sysfs(KSM_FP("merge_across_nodes"), merge_across_nodes)) 481 return KSFT_FAIL; 482 483 /* allocate 2 pages in 2 different NUMA nodes and fill them with the same data */ 484 first_node = get_first_mem_node(); 485 numa1_map_ptr = numa_alloc_onnode(page_size, first_node); 486 numa2_map_ptr = numa_alloc_onnode(page_size, get_next_mem_node(first_node)); 487 if (!numa1_map_ptr || !numa2_map_ptr) { 488 perror("numa_alloc_onnode"); 489 return KSFT_FAIL; 490 } 491 492 memset(numa1_map_ptr, '*', page_size); 493 memset(numa2_map_ptr, '*', page_size); 494 495 /* try to merge the pages */ 496 if (ksm_merge_pages(merge_type, numa1_map_ptr, page_size, start_time, timeout) || 497 ksm_merge_pages(merge_type, numa2_map_ptr, page_size, start_time, timeout)) 498 goto err_out; 499 500 /* 501 * verify that the right number of pages are merged: 502 * 1) if merge_across_nodes was enabled, 2 duplicate pages will be merged; 503 * 2) if merge_across_nodes = 0, there must be 0 merged pages, since there is 504 * only 1 unique page in each node and they can't be shared. 505 */ 506 if (merge_across_nodes && !assert_ksm_pages_count(page_count)) 507 goto err_out; 508 else if (!merge_across_nodes && !assert_ksm_pages_count(0)) 509 goto err_out; 510 511 numa_free(numa1_map_ptr, page_size); 512 numa_free(numa2_map_ptr, page_size); 513 printf("OK\n"); 514 return KSFT_PASS; 515 516 err_out: 517 numa_free(numa1_map_ptr, page_size); 518 numa_free(numa2_map_ptr, page_size); 519 printf("Not OK\n"); 520 return KSFT_FAIL; 521 } 522 523 static int ksm_merge_hugepages_time(int merge_type, int mapping, int prot, 524 int timeout, size_t map_size) 525 { 526 void *map_ptr, *map_ptr_orig; 527 struct timespec start_time, end_time; 528 unsigned long scan_time_ns; 529 int pagemap_fd, n_normal_pages, n_huge_pages; 530 531 if (!thp_is_enabled()) { 532 printf("Transparent Hugepages not available\n"); 533 return KSFT_SKIP; 534 } 535 536 map_size *= MB; 537 size_t len = map_size; 538 539 len -= len % HPAGE_SIZE; 540 map_ptr_orig = mmap(NULL, len + HPAGE_SIZE, PROT_READ | PROT_WRITE, 541 MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE, -1, 0); 542 map_ptr = map_ptr_orig + HPAGE_SIZE - (uintptr_t)map_ptr_orig % HPAGE_SIZE; 543 544 if (map_ptr_orig == MAP_FAILED) 545 err(2, "initial mmap"); 546 547 if (madvise(map_ptr, len, MADV_HUGEPAGE)) 548 err(2, "MADV_HUGEPAGE"); 549 550 pagemap_fd = open("/proc/self/pagemap", O_RDONLY); 551 if (pagemap_fd < 0) 552 err(2, "open pagemap"); 553 554 n_normal_pages = 0; 555 n_huge_pages = 0; 556 for (void *p = map_ptr; p < map_ptr + len; p += HPAGE_SIZE) { 557 if (allocate_transhuge(p, pagemap_fd) < 0) 558 n_normal_pages++; 559 else 560 n_huge_pages++; 561 } 562 printf("Number of normal pages: %d\n", n_normal_pages); 563 printf("Number of huge pages: %d\n", n_huge_pages); 564 565 memset(map_ptr, '*', len); 566 567 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 568 perror("clock_gettime"); 569 goto err_out; 570 } 571 if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout)) 572 goto err_out; 573 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { 574 perror("clock_gettime"); 575 goto err_out; 576 } 577 578 scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + 579 (end_time.tv_nsec - start_time.tv_nsec); 580 581 printf("Total size: %lu MiB\n", map_size / MB); 582 printf("Total time: %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC, 583 scan_time_ns % NSEC_PER_SEC); 584 printf("Average speed: %.3f MiB/s\n", (map_size / MB) / 585 ((double)scan_time_ns / NSEC_PER_SEC)); 586 587 munmap(map_ptr_orig, len + HPAGE_SIZE); 588 return KSFT_PASS; 589 590 err_out: 591 printf("Not OK\n"); 592 munmap(map_ptr_orig, len + HPAGE_SIZE); 593 return KSFT_FAIL; 594 } 595 596 static int ksm_merge_time(int merge_type, int mapping, int prot, int timeout, size_t map_size) 597 { 598 void *map_ptr; 599 struct timespec start_time, end_time; 600 unsigned long scan_time_ns; 601 602 map_size *= MB; 603 604 map_ptr = allocate_memory(NULL, prot, mapping, '*', map_size); 605 if (!map_ptr) 606 return KSFT_FAIL; 607 608 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 609 perror("clock_gettime"); 610 goto err_out; 611 } 612 if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout)) 613 goto err_out; 614 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { 615 perror("clock_gettime"); 616 goto err_out; 617 } 618 619 scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + 620 (end_time.tv_nsec - start_time.tv_nsec); 621 622 printf("Total size: %lu MiB\n", map_size / MB); 623 printf("Total time: %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC, 624 scan_time_ns % NSEC_PER_SEC); 625 printf("Average speed: %.3f MiB/s\n", (map_size / MB) / 626 ((double)scan_time_ns / NSEC_PER_SEC)); 627 628 munmap(map_ptr, map_size); 629 return KSFT_PASS; 630 631 err_out: 632 printf("Not OK\n"); 633 munmap(map_ptr, map_size); 634 return KSFT_FAIL; 635 } 636 637 static int ksm_unmerge_time(int merge_type, int mapping, int prot, int timeout, size_t map_size) 638 { 639 void *map_ptr; 640 struct timespec start_time, end_time; 641 unsigned long scan_time_ns; 642 643 map_size *= MB; 644 645 map_ptr = allocate_memory(NULL, prot, mapping, '*', map_size); 646 if (!map_ptr) 647 return KSFT_FAIL; 648 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 649 perror("clock_gettime"); 650 goto err_out; 651 } 652 if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout)) 653 goto err_out; 654 655 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 656 perror("clock_gettime"); 657 goto err_out; 658 } 659 if (ksm_unmerge_pages(map_ptr, map_size, start_time, timeout)) 660 goto err_out; 661 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { 662 perror("clock_gettime"); 663 goto err_out; 664 } 665 666 scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + 667 (end_time.tv_nsec - start_time.tv_nsec); 668 669 printf("Total size: %lu MiB\n", map_size / MB); 670 printf("Total time: %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC, 671 scan_time_ns % NSEC_PER_SEC); 672 printf("Average speed: %.3f MiB/s\n", (map_size / MB) / 673 ((double)scan_time_ns / NSEC_PER_SEC)); 674 675 munmap(map_ptr, map_size); 676 return KSFT_PASS; 677 678 err_out: 679 printf("Not OK\n"); 680 munmap(map_ptr, map_size); 681 return KSFT_FAIL; 682 } 683 684 static int ksm_cow_time(int merge_type, int mapping, int prot, int timeout, size_t page_size) 685 { 686 void *map_ptr; 687 struct timespec start_time, end_time; 688 unsigned long cow_time_ns; 689 690 /* page_count must be less than 2*page_size */ 691 size_t page_count = 4000; 692 693 map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count); 694 if (!map_ptr) 695 return KSFT_FAIL; 696 697 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 698 perror("clock_gettime"); 699 return KSFT_FAIL; 700 } 701 for (size_t i = 0; i < page_count - 1; i = i + 2) 702 memset(map_ptr + page_size * i, '-', 1); 703 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { 704 perror("clock_gettime"); 705 return KSFT_FAIL; 706 } 707 708 cow_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + 709 (end_time.tv_nsec - start_time.tv_nsec); 710 711 printf("Total size: %lu MiB\n\n", (page_size * page_count) / MB); 712 printf("Not merged pages:\n"); 713 printf("Total time: %ld.%09ld s\n", cow_time_ns / NSEC_PER_SEC, 714 cow_time_ns % NSEC_PER_SEC); 715 printf("Average speed: %.3f MiB/s\n\n", ((page_size * (page_count / 2)) / MB) / 716 ((double)cow_time_ns / NSEC_PER_SEC)); 717 718 /* Create 2000 pairs of duplicate pages */ 719 for (size_t i = 0; i < page_count - 1; i = i + 2) { 720 memset(map_ptr + page_size * i, '+', i / 2 + 1); 721 memset(map_ptr + page_size * (i + 1), '+', i / 2 + 1); 722 } 723 if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout)) 724 goto err_out; 725 726 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 727 perror("clock_gettime"); 728 goto err_out; 729 } 730 for (size_t i = 0; i < page_count - 1; i = i + 2) 731 memset(map_ptr + page_size * i, '-', 1); 732 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { 733 perror("clock_gettime"); 734 goto err_out; 735 } 736 737 cow_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + 738 (end_time.tv_nsec - start_time.tv_nsec); 739 740 printf("Merged pages:\n"); 741 printf("Total time: %ld.%09ld s\n", cow_time_ns / NSEC_PER_SEC, 742 cow_time_ns % NSEC_PER_SEC); 743 printf("Average speed: %.3f MiB/s\n", ((page_size * (page_count / 2)) / MB) / 744 ((double)cow_time_ns / NSEC_PER_SEC)); 745 746 munmap(map_ptr, page_size * page_count); 747 return KSFT_PASS; 748 749 err_out: 750 printf("Not OK\n"); 751 munmap(map_ptr, page_size * page_count); 752 return KSFT_FAIL; 753 } 754 755 int main(int argc, char *argv[]) 756 { 757 int ret = 0, opt; 758 int prot = 0; 759 int ksm_scan_limit_sec = KSM_SCAN_LIMIT_SEC_DEFAULT; 760 int merge_type = KSM_MERGE_TYPE_DEFAULT; 761 long page_count = KSM_PAGE_COUNT_DEFAULT; 762 size_t page_size = sysconf(_SC_PAGESIZE); 763 struct ksm_sysfs ksm_sysfs_old; 764 int test_name = CHECK_KSM_MERGE; 765 bool use_zero_pages = KSM_USE_ZERO_PAGES_DEFAULT; 766 bool merge_across_nodes = KSM_MERGE_ACROSS_NODES_DEFAULT; 767 long size_MB = 0; 768 769 while ((opt = getopt(argc, argv, "dha:p:l:z:m:s:t:MUZNPCHD")) != -1) { 770 switch (opt) { 771 case 'a': 772 prot = str_to_prot(optarg); 773 break; 774 case 'p': 775 page_count = atol(optarg); 776 if (page_count <= 0) { 777 printf("The number of pages must be greater than 0\n"); 778 return KSFT_FAIL; 779 } 780 break; 781 case 'l': 782 ksm_scan_limit_sec = atoi(optarg); 783 if (ksm_scan_limit_sec <= 0) { 784 printf("Timeout value must be greater than 0\n"); 785 return KSFT_FAIL; 786 } 787 break; 788 case 'h': 789 print_help(); 790 break; 791 case 'z': 792 if (strcmp(optarg, "0") == 0) 793 use_zero_pages = 0; 794 else 795 use_zero_pages = 1; 796 break; 797 case 'm': 798 if (strcmp(optarg, "0") == 0) 799 merge_across_nodes = 0; 800 else 801 merge_across_nodes = 1; 802 break; 803 case 'd': 804 debug = 1; 805 break; 806 case 's': 807 size_MB = atoi(optarg); 808 if (size_MB <= 0) { 809 printf("Size must be greater than 0\n"); 810 return KSFT_FAIL; 811 } 812 break; 813 case 't': 814 { 815 int tmp = atoi(optarg); 816 817 if (tmp < 0 || tmp > KSM_MERGE_LAST) { 818 printf("Invalid merge type\n"); 819 return KSFT_FAIL; 820 } 821 merge_type = tmp; 822 } 823 break; 824 case 'M': 825 break; 826 case 'U': 827 test_name = CHECK_KSM_UNMERGE; 828 break; 829 case 'Z': 830 test_name = CHECK_KSM_ZERO_PAGE_MERGE; 831 break; 832 case 'N': 833 test_name = CHECK_KSM_NUMA_MERGE; 834 break; 835 case 'P': 836 test_name = KSM_MERGE_TIME; 837 break; 838 case 'H': 839 test_name = KSM_MERGE_TIME_HUGE_PAGES; 840 break; 841 case 'D': 842 test_name = KSM_UNMERGE_TIME; 843 break; 844 case 'C': 845 test_name = KSM_COW_TIME; 846 break; 847 default: 848 return KSFT_FAIL; 849 } 850 } 851 852 if (prot == 0) 853 prot = str_to_prot(KSM_PROT_STR_DEFAULT); 854 855 if (access(KSM_SYSFS_PATH, F_OK)) { 856 printf("Config KSM not enabled\n"); 857 return KSFT_SKIP; 858 } 859 860 if (ksm_save_def(&ksm_sysfs_old)) { 861 printf("Cannot save default tunables\n"); 862 return KSFT_FAIL; 863 } 864 865 if (ksm_write_sysfs(KSM_FP("run"), 2) || 866 ksm_write_sysfs(KSM_FP("sleep_millisecs"), 0) || 867 numa_available() ? 0 : 868 ksm_write_sysfs(KSM_FP("merge_across_nodes"), 1) || 869 ksm_write_sysfs(KSM_FP("pages_to_scan"), page_count)) 870 return KSFT_FAIL; 871 872 switch (test_name) { 873 case CHECK_KSM_MERGE: 874 ret = check_ksm_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, page_count, 875 ksm_scan_limit_sec, page_size); 876 break; 877 case CHECK_KSM_UNMERGE: 878 ret = check_ksm_unmerge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, 879 ksm_scan_limit_sec, page_size); 880 break; 881 case CHECK_KSM_ZERO_PAGE_MERGE: 882 ret = check_ksm_zero_page_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, 883 page_count, ksm_scan_limit_sec, use_zero_pages, 884 page_size); 885 break; 886 case CHECK_KSM_NUMA_MERGE: 887 ret = check_ksm_numa_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, 888 ksm_scan_limit_sec, merge_across_nodes, page_size); 889 break; 890 case KSM_MERGE_TIME: 891 if (size_MB == 0) { 892 printf("Option '-s' is required.\n"); 893 return KSFT_FAIL; 894 } 895 ret = ksm_merge_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, 896 ksm_scan_limit_sec, size_MB); 897 break; 898 case KSM_MERGE_TIME_HUGE_PAGES: 899 if (size_MB == 0) { 900 printf("Option '-s' is required.\n"); 901 return KSFT_FAIL; 902 } 903 ret = ksm_merge_hugepages_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, 904 ksm_scan_limit_sec, size_MB); 905 break; 906 case KSM_UNMERGE_TIME: 907 if (size_MB == 0) { 908 printf("Option '-s' is required.\n"); 909 return KSFT_FAIL; 910 } 911 ret = ksm_unmerge_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, 912 ksm_scan_limit_sec, size_MB); 913 break; 914 case KSM_COW_TIME: 915 ret = ksm_cow_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, 916 ksm_scan_limit_sec, page_size); 917 break; 918 } 919 920 if (ksm_restore(&ksm_sysfs_old)) { 921 printf("Cannot restore default tunables\n"); 922 return KSFT_FAIL; 923 } 924 925 return ret; 926 } 927