1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <sys/mman.h> 4 #include <sys/prctl.h> 5 #include <sys/wait.h> 6 #include <stdbool.h> 7 #include <time.h> 8 #include <string.h> 9 #include <numa.h> 10 #include <unistd.h> 11 #include <fcntl.h> 12 #include <stdint.h> 13 #include <err.h> 14 15 #include "../kselftest.h" 16 #include <include/vdso/time64.h> 17 #include "vm_util.h" 18 19 #define KSM_SYSFS_PATH "/sys/kernel/mm/ksm/" 20 #define KSM_FP(s) (KSM_SYSFS_PATH s) 21 #define KSM_SCAN_LIMIT_SEC_DEFAULT 120 22 #define KSM_PAGE_COUNT_DEFAULT 10l 23 #define KSM_PROT_STR_DEFAULT "rw" 24 #define KSM_USE_ZERO_PAGES_DEFAULT false 25 #define KSM_MERGE_ACROSS_NODES_DEFAULT true 26 #define KSM_MERGE_TYPE_DEFAULT 0 27 #define MB (1ul << 20) 28 29 struct ksm_sysfs { 30 unsigned long max_page_sharing; 31 unsigned long merge_across_nodes; 32 unsigned long pages_to_scan; 33 unsigned long run; 34 unsigned long sleep_millisecs; 35 unsigned long stable_node_chains_prune_millisecs; 36 unsigned long use_zero_pages; 37 }; 38 39 enum ksm_merge_type { 40 KSM_MERGE_MADVISE, 41 KSM_MERGE_PRCTL, 42 KSM_MERGE_LAST = KSM_MERGE_PRCTL 43 }; 44 45 enum ksm_test_name { 46 CHECK_KSM_MERGE, 47 CHECK_KSM_UNMERGE, 48 CHECK_KSM_GET_MERGE_TYPE, 49 CHECK_KSM_ZERO_PAGE_MERGE, 50 CHECK_KSM_NUMA_MERGE, 51 KSM_MERGE_TIME, 52 KSM_MERGE_TIME_HUGE_PAGES, 53 KSM_UNMERGE_TIME, 54 KSM_COW_TIME 55 }; 56 57 int debug; 58 59 static int ksm_write_sysfs(const char *file_path, unsigned long val) 60 { 61 FILE *f = fopen(file_path, "w"); 62 63 if (!f) { 64 fprintf(stderr, "f %s\n", file_path); 65 perror("fopen"); 66 return 1; 67 } 68 if (fprintf(f, "%lu", val) < 0) { 69 perror("fprintf"); 70 fclose(f); 71 return 1; 72 } 73 fclose(f); 74 75 return 0; 76 } 77 78 static int ksm_read_sysfs(const char *file_path, unsigned long *val) 79 { 80 FILE *f = fopen(file_path, "r"); 81 82 if (!f) { 83 fprintf(stderr, "f %s\n", file_path); 84 perror("fopen"); 85 return 1; 86 } 87 if (fscanf(f, "%lu", val) != 1) { 88 perror("fscanf"); 89 fclose(f); 90 return 1; 91 } 92 fclose(f); 93 94 return 0; 95 } 96 97 static void ksm_print_sysfs(void) 98 { 99 unsigned long max_page_sharing, pages_sharing, pages_shared; 100 unsigned long full_scans, pages_unshared, pages_volatile; 101 unsigned long stable_node_chains, stable_node_dups; 102 long general_profit; 103 104 if (ksm_read_sysfs(KSM_FP("pages_shared"), &pages_shared) || 105 ksm_read_sysfs(KSM_FP("pages_sharing"), &pages_sharing) || 106 ksm_read_sysfs(KSM_FP("max_page_sharing"), &max_page_sharing) || 107 ksm_read_sysfs(KSM_FP("full_scans"), &full_scans) || 108 ksm_read_sysfs(KSM_FP("pages_unshared"), &pages_unshared) || 109 ksm_read_sysfs(KSM_FP("pages_volatile"), &pages_volatile) || 110 ksm_read_sysfs(KSM_FP("stable_node_chains"), &stable_node_chains) || 111 ksm_read_sysfs(KSM_FP("stable_node_dups"), &stable_node_dups) || 112 ksm_read_sysfs(KSM_FP("general_profit"), (unsigned long *)&general_profit)) 113 return; 114 115 printf("pages_shared : %lu\n", pages_shared); 116 printf("pages_sharing : %lu\n", pages_sharing); 117 printf("max_page_sharing : %lu\n", max_page_sharing); 118 printf("full_scans : %lu\n", full_scans); 119 printf("pages_unshared : %lu\n", pages_unshared); 120 printf("pages_volatile : %lu\n", pages_volatile); 121 printf("stable_node_chains: %lu\n", stable_node_chains); 122 printf("stable_node_dups : %lu\n", stable_node_dups); 123 printf("general_profit : %ld\n", general_profit); 124 } 125 126 static void ksm_print_procfs(void) 127 { 128 const char *file_name = "/proc/self/ksm_stat"; 129 char buffer[512]; 130 FILE *f = fopen(file_name, "r"); 131 132 if (!f) { 133 fprintf(stderr, "f %s\n", file_name); 134 perror("fopen"); 135 return; 136 } 137 138 while (fgets(buffer, sizeof(buffer), f)) 139 printf("%s", buffer); 140 141 fclose(f); 142 } 143 144 static int str_to_prot(char *prot_str) 145 { 146 int prot = 0; 147 148 if ((strchr(prot_str, 'r')) != NULL) 149 prot |= PROT_READ; 150 if ((strchr(prot_str, 'w')) != NULL) 151 prot |= PROT_WRITE; 152 if ((strchr(prot_str, 'x')) != NULL) 153 prot |= PROT_EXEC; 154 155 return prot; 156 } 157 158 static void print_help(void) 159 { 160 printf("usage: ksm_tests [-h] <test type> [-a prot] [-p page_count] [-l timeout]\n" 161 "[-z use_zero_pages] [-m merge_across_nodes] [-s size]\n"); 162 163 printf("Supported <test type>:\n" 164 " -M (page merging)\n" 165 " -Z (zero pages merging)\n" 166 " -N (merging of pages in different NUMA nodes)\n" 167 " -U (page unmerging)\n" 168 " -P evaluate merging time and speed.\n" 169 " For this test, the size of duplicated memory area (in MiB)\n" 170 " must be provided using -s option\n" 171 " -H evaluate merging time and speed of area allocated mostly with huge pages\n" 172 " For this test, the size of duplicated memory area (in MiB)\n" 173 " must be provided using -s option\n" 174 " -D evaluate unmerging time and speed when disabling KSM.\n" 175 " For this test, the size of duplicated memory area (in MiB)\n" 176 " must be provided using -s option\n" 177 " -C evaluate the time required to break COW of merged pages.\n\n"); 178 179 printf(" -a: specify the access protections of pages.\n" 180 " <prot> must be of the form [rwx].\n" 181 " Default: %s\n", KSM_PROT_STR_DEFAULT); 182 printf(" -p: specify the number of pages to test.\n" 183 " Default: %ld\n", KSM_PAGE_COUNT_DEFAULT); 184 printf(" -l: limit the maximum running time (in seconds) for a test.\n" 185 " Default: %d seconds\n", KSM_SCAN_LIMIT_SEC_DEFAULT); 186 printf(" -z: change use_zero_pages tunable\n" 187 " Default: %d\n", KSM_USE_ZERO_PAGES_DEFAULT); 188 printf(" -m: change merge_across_nodes tunable\n" 189 " Default: %d\n", KSM_MERGE_ACROSS_NODES_DEFAULT); 190 printf(" -d: turn debugging output on\n"); 191 printf(" -s: the size of duplicated memory area (in MiB)\n"); 192 printf(" -t: KSM merge type\n" 193 " Default: 0\n" 194 " 0: madvise merging\n" 195 " 1: prctl merging\n"); 196 197 exit(0); 198 } 199 200 static void *allocate_memory(void *ptr, int prot, int mapping, char data, size_t map_size) 201 { 202 void *map_ptr = mmap(ptr, map_size, PROT_WRITE, mapping, -1, 0); 203 204 if (!map_ptr) { 205 perror("mmap"); 206 return NULL; 207 } 208 memset(map_ptr, data, map_size); 209 if (mprotect(map_ptr, map_size, prot)) { 210 perror("mprotect"); 211 munmap(map_ptr, map_size); 212 return NULL; 213 } 214 215 return map_ptr; 216 } 217 218 static int ksm_do_scan(int scan_count, struct timespec start_time, int timeout) 219 { 220 struct timespec cur_time; 221 unsigned long cur_scan, init_scan; 222 223 if (ksm_read_sysfs(KSM_FP("full_scans"), &init_scan)) 224 return 1; 225 cur_scan = init_scan; 226 227 while (cur_scan < init_scan + scan_count) { 228 if (ksm_read_sysfs(KSM_FP("full_scans"), &cur_scan)) 229 return 1; 230 if (clock_gettime(CLOCK_MONOTONIC_RAW, &cur_time)) { 231 perror("clock_gettime"); 232 return 1; 233 } 234 if ((cur_time.tv_sec - start_time.tv_sec) > timeout) { 235 printf("Scan time limit exceeded\n"); 236 return 1; 237 } 238 } 239 240 return 0; 241 } 242 243 static int ksm_merge_pages(int merge_type, void *addr, size_t size, 244 struct timespec start_time, int timeout) 245 { 246 if (merge_type == KSM_MERGE_MADVISE) { 247 if (madvise(addr, size, MADV_MERGEABLE)) { 248 perror("madvise"); 249 return 1; 250 } 251 } else if (merge_type == KSM_MERGE_PRCTL) { 252 if (prctl(PR_SET_MEMORY_MERGE, 1, 0, 0, 0)) { 253 perror("prctl"); 254 return 1; 255 } 256 } 257 258 if (ksm_write_sysfs(KSM_FP("run"), 1)) 259 return 1; 260 261 /* Since merging occurs only after 2 scans, make sure to get at least 2 full scans */ 262 if (ksm_do_scan(2, start_time, timeout)) 263 return 1; 264 265 return 0; 266 } 267 268 static int ksm_unmerge_pages(void *addr, size_t size, 269 struct timespec start_time, int timeout) 270 { 271 if (madvise(addr, size, MADV_UNMERGEABLE)) { 272 perror("madvise"); 273 return 1; 274 } 275 return 0; 276 } 277 278 static bool assert_ksm_pages_count(long dupl_page_count) 279 { 280 unsigned long max_page_sharing, pages_sharing, pages_shared; 281 282 if (ksm_read_sysfs(KSM_FP("pages_shared"), &pages_shared) || 283 ksm_read_sysfs(KSM_FP("pages_sharing"), &pages_sharing) || 284 ksm_read_sysfs(KSM_FP("max_page_sharing"), &max_page_sharing)) 285 return false; 286 287 if (debug) { 288 ksm_print_sysfs(); 289 ksm_print_procfs(); 290 } 291 292 /* 293 * Since there must be at least 2 pages for merging and 1 page can be 294 * shared with the limited number of pages (max_page_sharing), sometimes 295 * there are 'leftover' pages that cannot be merged. For example, if there 296 * are 11 pages and max_page_sharing = 10, then only 10 pages will be 297 * merged and the 11th page won't be affected. As a result, when the number 298 * of duplicate pages is divided by max_page_sharing and the remainder is 1, 299 * pages_shared and pages_sharing values will be equal between dupl_page_count 300 * and dupl_page_count - 1. 301 */ 302 if (dupl_page_count % max_page_sharing == 1 || dupl_page_count % max_page_sharing == 0) { 303 if (pages_shared == dupl_page_count / max_page_sharing && 304 pages_sharing == pages_shared * (max_page_sharing - 1)) 305 return true; 306 } else { 307 if (pages_shared == (dupl_page_count / max_page_sharing + 1) && 308 pages_sharing == dupl_page_count - pages_shared) 309 return true; 310 } 311 312 return false; 313 } 314 315 static int ksm_save_def(struct ksm_sysfs *ksm_sysfs) 316 { 317 if (ksm_read_sysfs(KSM_FP("max_page_sharing"), &ksm_sysfs->max_page_sharing) || 318 numa_available() ? 0 : 319 ksm_read_sysfs(KSM_FP("merge_across_nodes"), &ksm_sysfs->merge_across_nodes) || 320 ksm_read_sysfs(KSM_FP("sleep_millisecs"), &ksm_sysfs->sleep_millisecs) || 321 ksm_read_sysfs(KSM_FP("pages_to_scan"), &ksm_sysfs->pages_to_scan) || 322 ksm_read_sysfs(KSM_FP("run"), &ksm_sysfs->run) || 323 ksm_read_sysfs(KSM_FP("stable_node_chains_prune_millisecs"), 324 &ksm_sysfs->stable_node_chains_prune_millisecs) || 325 ksm_read_sysfs(KSM_FP("use_zero_pages"), &ksm_sysfs->use_zero_pages)) 326 return 1; 327 328 return 0; 329 } 330 331 static int ksm_restore(struct ksm_sysfs *ksm_sysfs) 332 { 333 if (ksm_write_sysfs(KSM_FP("max_page_sharing"), ksm_sysfs->max_page_sharing) || 334 numa_available() ? 0 : 335 ksm_write_sysfs(KSM_FP("merge_across_nodes"), ksm_sysfs->merge_across_nodes) || 336 ksm_write_sysfs(KSM_FP("pages_to_scan"), ksm_sysfs->pages_to_scan) || 337 ksm_write_sysfs(KSM_FP("run"), ksm_sysfs->run) || 338 ksm_write_sysfs(KSM_FP("sleep_millisecs"), ksm_sysfs->sleep_millisecs) || 339 ksm_write_sysfs(KSM_FP("stable_node_chains_prune_millisecs"), 340 ksm_sysfs->stable_node_chains_prune_millisecs) || 341 ksm_write_sysfs(KSM_FP("use_zero_pages"), ksm_sysfs->use_zero_pages)) 342 return 1; 343 344 return 0; 345 } 346 347 static int check_ksm_merge(int merge_type, int mapping, int prot, 348 long page_count, int timeout, size_t page_size) 349 { 350 void *map_ptr; 351 struct timespec start_time; 352 353 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 354 perror("clock_gettime"); 355 return KSFT_FAIL; 356 } 357 358 /* fill pages with the same data and merge them */ 359 map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count); 360 if (!map_ptr) 361 return KSFT_FAIL; 362 363 if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout)) 364 goto err_out; 365 366 /* verify that the right number of pages are merged */ 367 if (assert_ksm_pages_count(page_count)) { 368 printf("OK\n"); 369 munmap(map_ptr, page_size * page_count); 370 if (merge_type == KSM_MERGE_PRCTL) 371 prctl(PR_SET_MEMORY_MERGE, 0, 0, 0, 0); 372 return KSFT_PASS; 373 } 374 375 err_out: 376 printf("Not OK\n"); 377 munmap(map_ptr, page_size * page_count); 378 return KSFT_FAIL; 379 } 380 381 static int check_ksm_unmerge(int merge_type, int mapping, int prot, int timeout, size_t page_size) 382 { 383 void *map_ptr; 384 struct timespec start_time; 385 int page_count = 2; 386 387 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 388 perror("clock_gettime"); 389 return KSFT_FAIL; 390 } 391 392 /* fill pages with the same data and merge them */ 393 map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count); 394 if (!map_ptr) 395 return KSFT_FAIL; 396 397 if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout)) 398 goto err_out; 399 400 /* change 1 byte in each of the 2 pages -- KSM must automatically unmerge them */ 401 memset(map_ptr, '-', 1); 402 memset(map_ptr + page_size, '+', 1); 403 404 /* get at least 1 scan, so KSM can detect that the pages were modified */ 405 if (ksm_do_scan(1, start_time, timeout)) 406 goto err_out; 407 408 /* check that unmerging was successful and 0 pages are currently merged */ 409 if (assert_ksm_pages_count(0)) { 410 printf("OK\n"); 411 munmap(map_ptr, page_size * page_count); 412 return KSFT_PASS; 413 } 414 415 err_out: 416 printf("Not OK\n"); 417 munmap(map_ptr, page_size * page_count); 418 return KSFT_FAIL; 419 } 420 421 static int check_ksm_zero_page_merge(int merge_type, int mapping, int prot, long page_count, 422 int timeout, bool use_zero_pages, size_t page_size) 423 { 424 void *map_ptr; 425 struct timespec start_time; 426 427 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 428 perror("clock_gettime"); 429 return KSFT_FAIL; 430 } 431 432 if (ksm_write_sysfs(KSM_FP("use_zero_pages"), use_zero_pages)) 433 return KSFT_FAIL; 434 435 /* fill pages with zero and try to merge them */ 436 map_ptr = allocate_memory(NULL, prot, mapping, 0, page_size * page_count); 437 if (!map_ptr) 438 return KSFT_FAIL; 439 440 if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout)) 441 goto err_out; 442 443 /* 444 * verify that the right number of pages are merged: 445 * 1) if use_zero_pages is set to 1, empty pages are merged 446 * with the kernel zero page instead of with each other; 447 * 2) if use_zero_pages is set to 0, empty pages are not treated specially 448 * and merged as usual. 449 */ 450 if (use_zero_pages && !assert_ksm_pages_count(0)) 451 goto err_out; 452 else if (!use_zero_pages && !assert_ksm_pages_count(page_count)) 453 goto err_out; 454 455 printf("OK\n"); 456 munmap(map_ptr, page_size * page_count); 457 return KSFT_PASS; 458 459 err_out: 460 printf("Not OK\n"); 461 munmap(map_ptr, page_size * page_count); 462 return KSFT_FAIL; 463 } 464 465 static int get_next_mem_node(int node) 466 { 467 468 long node_size; 469 int mem_node = 0; 470 int i, max_node = numa_max_node(); 471 472 for (i = node + 1; i <= max_node + node; i++) { 473 mem_node = i % (max_node + 1); 474 node_size = numa_node_size(mem_node, NULL); 475 if (node_size > 0) 476 break; 477 } 478 return mem_node; 479 } 480 481 static int get_first_mem_node(void) 482 { 483 return get_next_mem_node(numa_max_node()); 484 } 485 486 static int check_ksm_numa_merge(int merge_type, int mapping, int prot, int timeout, 487 bool merge_across_nodes, size_t page_size) 488 { 489 void *numa1_map_ptr, *numa2_map_ptr; 490 struct timespec start_time; 491 int page_count = 2; 492 int first_node; 493 494 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 495 perror("clock_gettime"); 496 return KSFT_FAIL; 497 } 498 499 if (numa_available() < 0) { 500 perror("NUMA support not enabled"); 501 return KSFT_SKIP; 502 } 503 if (numa_num_configured_nodes() <= 1) { 504 printf("At least 2 NUMA nodes must be available\n"); 505 return KSFT_SKIP; 506 } 507 if (ksm_write_sysfs(KSM_FP("merge_across_nodes"), merge_across_nodes)) 508 return KSFT_FAIL; 509 510 /* allocate 2 pages in 2 different NUMA nodes and fill them with the same data */ 511 first_node = get_first_mem_node(); 512 numa1_map_ptr = numa_alloc_onnode(page_size, first_node); 513 numa2_map_ptr = numa_alloc_onnode(page_size, get_next_mem_node(first_node)); 514 if (!numa1_map_ptr || !numa2_map_ptr) { 515 perror("numa_alloc_onnode"); 516 return KSFT_FAIL; 517 } 518 519 memset(numa1_map_ptr, '*', page_size); 520 memset(numa2_map_ptr, '*', page_size); 521 522 /* try to merge the pages */ 523 if (ksm_merge_pages(merge_type, numa1_map_ptr, page_size, start_time, timeout) || 524 ksm_merge_pages(merge_type, numa2_map_ptr, page_size, start_time, timeout)) 525 goto err_out; 526 527 /* 528 * verify that the right number of pages are merged: 529 * 1) if merge_across_nodes was enabled, 2 duplicate pages will be merged; 530 * 2) if merge_across_nodes = 0, there must be 0 merged pages, since there is 531 * only 1 unique page in each node and they can't be shared. 532 */ 533 if (merge_across_nodes && !assert_ksm_pages_count(page_count)) 534 goto err_out; 535 else if (!merge_across_nodes && !assert_ksm_pages_count(0)) 536 goto err_out; 537 538 numa_free(numa1_map_ptr, page_size); 539 numa_free(numa2_map_ptr, page_size); 540 printf("OK\n"); 541 return KSFT_PASS; 542 543 err_out: 544 numa_free(numa1_map_ptr, page_size); 545 numa_free(numa2_map_ptr, page_size); 546 printf("Not OK\n"); 547 return KSFT_FAIL; 548 } 549 550 static int ksm_merge_hugepages_time(int merge_type, int mapping, int prot, 551 int timeout, size_t map_size) 552 { 553 void *map_ptr, *map_ptr_orig; 554 struct timespec start_time, end_time; 555 unsigned long scan_time_ns; 556 int pagemap_fd, n_normal_pages, n_huge_pages; 557 558 map_size *= MB; 559 size_t len = map_size; 560 561 len -= len % HPAGE_SIZE; 562 map_ptr_orig = mmap(NULL, len + HPAGE_SIZE, PROT_READ | PROT_WRITE, 563 MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE, -1, 0); 564 map_ptr = map_ptr_orig + HPAGE_SIZE - (uintptr_t)map_ptr_orig % HPAGE_SIZE; 565 566 if (map_ptr_orig == MAP_FAILED) 567 err(2, "initial mmap"); 568 569 if (madvise(map_ptr, len, MADV_HUGEPAGE)) 570 err(2, "MADV_HUGEPAGE"); 571 572 pagemap_fd = open("/proc/self/pagemap", O_RDONLY); 573 if (pagemap_fd < 0) 574 err(2, "open pagemap"); 575 576 n_normal_pages = 0; 577 n_huge_pages = 0; 578 for (void *p = map_ptr; p < map_ptr + len; p += HPAGE_SIZE) { 579 if (allocate_transhuge(p, pagemap_fd) < 0) 580 n_normal_pages++; 581 else 582 n_huge_pages++; 583 } 584 printf("Number of normal pages: %d\n", n_normal_pages); 585 printf("Number of huge pages: %d\n", n_huge_pages); 586 587 memset(map_ptr, '*', len); 588 589 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 590 perror("clock_gettime"); 591 goto err_out; 592 } 593 if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout)) 594 goto err_out; 595 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { 596 perror("clock_gettime"); 597 goto err_out; 598 } 599 600 scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + 601 (end_time.tv_nsec - start_time.tv_nsec); 602 603 printf("Total size: %lu MiB\n", map_size / MB); 604 printf("Total time: %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC, 605 scan_time_ns % NSEC_PER_SEC); 606 printf("Average speed: %.3f MiB/s\n", (map_size / MB) / 607 ((double)scan_time_ns / NSEC_PER_SEC)); 608 609 munmap(map_ptr_orig, len + HPAGE_SIZE); 610 return KSFT_PASS; 611 612 err_out: 613 printf("Not OK\n"); 614 munmap(map_ptr_orig, len + HPAGE_SIZE); 615 return KSFT_FAIL; 616 } 617 618 static int ksm_merge_time(int merge_type, int mapping, int prot, int timeout, size_t map_size) 619 { 620 void *map_ptr; 621 struct timespec start_time, end_time; 622 unsigned long scan_time_ns; 623 624 map_size *= MB; 625 626 map_ptr = allocate_memory(NULL, prot, mapping, '*', map_size); 627 if (!map_ptr) 628 return KSFT_FAIL; 629 630 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 631 perror("clock_gettime"); 632 goto err_out; 633 } 634 if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout)) 635 goto err_out; 636 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { 637 perror("clock_gettime"); 638 goto err_out; 639 } 640 641 scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + 642 (end_time.tv_nsec - start_time.tv_nsec); 643 644 printf("Total size: %lu MiB\n", map_size / MB); 645 printf("Total time: %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC, 646 scan_time_ns % NSEC_PER_SEC); 647 printf("Average speed: %.3f MiB/s\n", (map_size / MB) / 648 ((double)scan_time_ns / NSEC_PER_SEC)); 649 650 munmap(map_ptr, map_size); 651 return KSFT_PASS; 652 653 err_out: 654 printf("Not OK\n"); 655 munmap(map_ptr, map_size); 656 return KSFT_FAIL; 657 } 658 659 static int ksm_unmerge_time(int merge_type, int mapping, int prot, int timeout, size_t map_size) 660 { 661 void *map_ptr; 662 struct timespec start_time, end_time; 663 unsigned long scan_time_ns; 664 665 map_size *= MB; 666 667 map_ptr = allocate_memory(NULL, prot, mapping, '*', map_size); 668 if (!map_ptr) 669 return KSFT_FAIL; 670 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 671 perror("clock_gettime"); 672 goto err_out; 673 } 674 if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout)) 675 goto err_out; 676 677 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 678 perror("clock_gettime"); 679 goto err_out; 680 } 681 if (ksm_unmerge_pages(map_ptr, map_size, start_time, timeout)) 682 goto err_out; 683 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { 684 perror("clock_gettime"); 685 goto err_out; 686 } 687 688 scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + 689 (end_time.tv_nsec - start_time.tv_nsec); 690 691 printf("Total size: %lu MiB\n", map_size / MB); 692 printf("Total time: %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC, 693 scan_time_ns % NSEC_PER_SEC); 694 printf("Average speed: %.3f MiB/s\n", (map_size / MB) / 695 ((double)scan_time_ns / NSEC_PER_SEC)); 696 697 munmap(map_ptr, map_size); 698 return KSFT_PASS; 699 700 err_out: 701 printf("Not OK\n"); 702 munmap(map_ptr, map_size); 703 return KSFT_FAIL; 704 } 705 706 static int ksm_cow_time(int merge_type, int mapping, int prot, int timeout, size_t page_size) 707 { 708 void *map_ptr; 709 struct timespec start_time, end_time; 710 unsigned long cow_time_ns; 711 712 /* page_count must be less than 2*page_size */ 713 size_t page_count = 4000; 714 715 map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count); 716 if (!map_ptr) 717 return KSFT_FAIL; 718 719 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 720 perror("clock_gettime"); 721 return KSFT_FAIL; 722 } 723 for (size_t i = 0; i < page_count - 1; i = i + 2) 724 memset(map_ptr + page_size * i, '-', 1); 725 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { 726 perror("clock_gettime"); 727 return KSFT_FAIL; 728 } 729 730 cow_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + 731 (end_time.tv_nsec - start_time.tv_nsec); 732 733 printf("Total size: %lu MiB\n\n", (page_size * page_count) / MB); 734 printf("Not merged pages:\n"); 735 printf("Total time: %ld.%09ld s\n", cow_time_ns / NSEC_PER_SEC, 736 cow_time_ns % NSEC_PER_SEC); 737 printf("Average speed: %.3f MiB/s\n\n", ((page_size * (page_count / 2)) / MB) / 738 ((double)cow_time_ns / NSEC_PER_SEC)); 739 740 /* Create 2000 pairs of duplicate pages */ 741 for (size_t i = 0; i < page_count - 1; i = i + 2) { 742 memset(map_ptr + page_size * i, '+', i / 2 + 1); 743 memset(map_ptr + page_size * (i + 1), '+', i / 2 + 1); 744 } 745 if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout)) 746 goto err_out; 747 748 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 749 perror("clock_gettime"); 750 goto err_out; 751 } 752 for (size_t i = 0; i < page_count - 1; i = i + 2) 753 memset(map_ptr + page_size * i, '-', 1); 754 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { 755 perror("clock_gettime"); 756 goto err_out; 757 } 758 759 cow_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + 760 (end_time.tv_nsec - start_time.tv_nsec); 761 762 printf("Merged pages:\n"); 763 printf("Total time: %ld.%09ld s\n", cow_time_ns / NSEC_PER_SEC, 764 cow_time_ns % NSEC_PER_SEC); 765 printf("Average speed: %.3f MiB/s\n", ((page_size * (page_count / 2)) / MB) / 766 ((double)cow_time_ns / NSEC_PER_SEC)); 767 768 munmap(map_ptr, page_size * page_count); 769 return KSFT_PASS; 770 771 err_out: 772 printf("Not OK\n"); 773 munmap(map_ptr, page_size * page_count); 774 return KSFT_FAIL; 775 } 776 777 int main(int argc, char *argv[]) 778 { 779 int ret, opt; 780 int prot = 0; 781 int ksm_scan_limit_sec = KSM_SCAN_LIMIT_SEC_DEFAULT; 782 int merge_type = KSM_MERGE_TYPE_DEFAULT; 783 long page_count = KSM_PAGE_COUNT_DEFAULT; 784 size_t page_size = sysconf(_SC_PAGESIZE); 785 struct ksm_sysfs ksm_sysfs_old; 786 int test_name = CHECK_KSM_MERGE; 787 bool use_zero_pages = KSM_USE_ZERO_PAGES_DEFAULT; 788 bool merge_across_nodes = KSM_MERGE_ACROSS_NODES_DEFAULT; 789 long size_MB = 0; 790 791 while ((opt = getopt(argc, argv, "dha:p:l:z:m:s:t:MUZNPCHD")) != -1) { 792 switch (opt) { 793 case 'a': 794 prot = str_to_prot(optarg); 795 break; 796 case 'p': 797 page_count = atol(optarg); 798 if (page_count <= 0) { 799 printf("The number of pages must be greater than 0\n"); 800 return KSFT_FAIL; 801 } 802 break; 803 case 'l': 804 ksm_scan_limit_sec = atoi(optarg); 805 if (ksm_scan_limit_sec <= 0) { 806 printf("Timeout value must be greater than 0\n"); 807 return KSFT_FAIL; 808 } 809 break; 810 case 'h': 811 print_help(); 812 break; 813 case 'z': 814 if (strcmp(optarg, "0") == 0) 815 use_zero_pages = 0; 816 else 817 use_zero_pages = 1; 818 break; 819 case 'm': 820 if (strcmp(optarg, "0") == 0) 821 merge_across_nodes = 0; 822 else 823 merge_across_nodes = 1; 824 break; 825 case 'd': 826 debug = 1; 827 break; 828 case 's': 829 size_MB = atoi(optarg); 830 if (size_MB <= 0) { 831 printf("Size must be greater than 0\n"); 832 return KSFT_FAIL; 833 } 834 break; 835 case 't': 836 { 837 int tmp = atoi(optarg); 838 839 if (tmp < 0 || tmp > KSM_MERGE_LAST) { 840 printf("Invalid merge type\n"); 841 return KSFT_FAIL; 842 } 843 merge_type = tmp; 844 } 845 break; 846 case 'M': 847 break; 848 case 'U': 849 test_name = CHECK_KSM_UNMERGE; 850 break; 851 case 'Z': 852 test_name = CHECK_KSM_ZERO_PAGE_MERGE; 853 break; 854 case 'N': 855 test_name = CHECK_KSM_NUMA_MERGE; 856 break; 857 case 'P': 858 test_name = KSM_MERGE_TIME; 859 break; 860 case 'H': 861 test_name = KSM_MERGE_TIME_HUGE_PAGES; 862 break; 863 case 'D': 864 test_name = KSM_UNMERGE_TIME; 865 break; 866 case 'C': 867 test_name = KSM_COW_TIME; 868 break; 869 default: 870 return KSFT_FAIL; 871 } 872 } 873 874 if (prot == 0) 875 prot = str_to_prot(KSM_PROT_STR_DEFAULT); 876 877 if (access(KSM_SYSFS_PATH, F_OK)) { 878 printf("Config KSM not enabled\n"); 879 return KSFT_SKIP; 880 } 881 882 if (ksm_save_def(&ksm_sysfs_old)) { 883 printf("Cannot save default tunables\n"); 884 return KSFT_FAIL; 885 } 886 887 if (ksm_write_sysfs(KSM_FP("run"), 2) || 888 ksm_write_sysfs(KSM_FP("sleep_millisecs"), 0) || 889 numa_available() ? 0 : 890 ksm_write_sysfs(KSM_FP("merge_across_nodes"), 1) || 891 ksm_write_sysfs(KSM_FP("pages_to_scan"), page_count)) 892 return KSFT_FAIL; 893 894 switch (test_name) { 895 case CHECK_KSM_MERGE: 896 ret = check_ksm_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, page_count, 897 ksm_scan_limit_sec, page_size); 898 break; 899 case CHECK_KSM_UNMERGE: 900 ret = check_ksm_unmerge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, 901 ksm_scan_limit_sec, page_size); 902 break; 903 case CHECK_KSM_ZERO_PAGE_MERGE: 904 ret = check_ksm_zero_page_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, 905 page_count, ksm_scan_limit_sec, use_zero_pages, 906 page_size); 907 break; 908 case CHECK_KSM_NUMA_MERGE: 909 ret = check_ksm_numa_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, 910 ksm_scan_limit_sec, merge_across_nodes, page_size); 911 break; 912 case KSM_MERGE_TIME: 913 if (size_MB == 0) { 914 printf("Option '-s' is required.\n"); 915 return KSFT_FAIL; 916 } 917 ret = ksm_merge_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, 918 ksm_scan_limit_sec, size_MB); 919 break; 920 case KSM_MERGE_TIME_HUGE_PAGES: 921 if (size_MB == 0) { 922 printf("Option '-s' is required.\n"); 923 return KSFT_FAIL; 924 } 925 ret = ksm_merge_hugepages_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, 926 ksm_scan_limit_sec, size_MB); 927 break; 928 case KSM_UNMERGE_TIME: 929 if (size_MB == 0) { 930 printf("Option '-s' is required.\n"); 931 return KSFT_FAIL; 932 } 933 ret = ksm_unmerge_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, 934 ksm_scan_limit_sec, size_MB); 935 break; 936 case KSM_COW_TIME: 937 ret = ksm_cow_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, 938 ksm_scan_limit_sec, page_size); 939 break; 940 } 941 942 if (ksm_restore(&ksm_sysfs_old)) { 943 printf("Cannot restore default tunables\n"); 944 return KSFT_FAIL; 945 } 946 947 return ret; 948 } 949