1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <sys/mman.h> 4 #include <sys/prctl.h> 5 #include <sys/wait.h> 6 #include <stdbool.h> 7 #include <time.h> 8 #include <string.h> 9 #include <numa.h> 10 #include <unistd.h> 11 #include <fcntl.h> 12 #include <stdint.h> 13 #include <err.h> 14 15 #include "kselftest.h" 16 #include <include/vdso/time64.h> 17 #include "vm_util.h" 18 #include "hugepage_settings.h" 19 20 #define KSM_SYSFS_PATH "/sys/kernel/mm/ksm/" 21 #define KSM_FP(s) (KSM_SYSFS_PATH s) 22 #define KSM_SCAN_LIMIT_SEC_DEFAULT 120 23 #define KSM_PAGE_COUNT_DEFAULT 10l 24 #define KSM_PROT_STR_DEFAULT "rw" 25 #define KSM_USE_ZERO_PAGES_DEFAULT false 26 #define KSM_MERGE_ACROSS_NODES_DEFAULT true 27 #define KSM_MERGE_TYPE_DEFAULT 0 28 #define MB (1ul << 20) 29 30 struct ksm_sysfs { 31 unsigned long max_page_sharing; 32 unsigned long merge_across_nodes; 33 unsigned long pages_to_scan; 34 unsigned long run; 35 unsigned long sleep_millisecs; 36 unsigned long stable_node_chains_prune_millisecs; 37 unsigned long use_zero_pages; 38 }; 39 40 enum ksm_merge_type { 41 KSM_MERGE_MADVISE, 42 KSM_MERGE_PRCTL, 43 KSM_MERGE_LAST = KSM_MERGE_PRCTL 44 }; 45 46 enum ksm_test_name { 47 CHECK_KSM_MERGE, 48 CHECK_KSM_UNMERGE, 49 CHECK_KSM_GET_MERGE_TYPE, 50 CHECK_KSM_ZERO_PAGE_MERGE, 51 CHECK_KSM_NUMA_MERGE, 52 KSM_MERGE_TIME, 53 KSM_MERGE_TIME_HUGE_PAGES, 54 KSM_UNMERGE_TIME, 55 KSM_COW_TIME 56 }; 57 58 int debug; 59 60 static int ksm_write_sysfs(const char *file_path, unsigned long val) 61 { 62 return write_sysfs(file_path, val); 63 } 64 65 static int ksm_read_sysfs(const char *file_path, unsigned long *val) 66 { 67 return read_sysfs(file_path, val); 68 } 69 70 static void ksm_print_sysfs(void) 71 { 72 unsigned long max_page_sharing, pages_sharing, pages_shared; 73 unsigned long full_scans, pages_unshared, pages_volatile; 74 unsigned long stable_node_chains, stable_node_dups; 75 long general_profit; 76 77 if (ksm_read_sysfs(KSM_FP("pages_shared"), &pages_shared) || 78 ksm_read_sysfs(KSM_FP("pages_sharing"), &pages_sharing) || 79 ksm_read_sysfs(KSM_FP("max_page_sharing"), &max_page_sharing) || 80 ksm_read_sysfs(KSM_FP("full_scans"), &full_scans) || 81 ksm_read_sysfs(KSM_FP("pages_unshared"), &pages_unshared) || 82 ksm_read_sysfs(KSM_FP("pages_volatile"), &pages_volatile) || 83 ksm_read_sysfs(KSM_FP("stable_node_chains"), &stable_node_chains) || 84 ksm_read_sysfs(KSM_FP("stable_node_dups"), &stable_node_dups) || 85 ksm_read_sysfs(KSM_FP("general_profit"), (unsigned long *)&general_profit)) 86 return; 87 88 printf("pages_shared : %lu\n", pages_shared); 89 printf("pages_sharing : %lu\n", pages_sharing); 90 printf("max_page_sharing : %lu\n", max_page_sharing); 91 printf("full_scans : %lu\n", full_scans); 92 printf("pages_unshared : %lu\n", pages_unshared); 93 printf("pages_volatile : %lu\n", pages_volatile); 94 printf("stable_node_chains: %lu\n", stable_node_chains); 95 printf("stable_node_dups : %lu\n", stable_node_dups); 96 printf("general_profit : %ld\n", general_profit); 97 } 98 99 static void ksm_print_procfs(void) 100 { 101 const char *file_name = "/proc/self/ksm_stat"; 102 char buffer[512]; 103 FILE *f = fopen(file_name, "r"); 104 105 if (!f) { 106 fprintf(stderr, "f %s\n", file_name); 107 perror("fopen"); 108 return; 109 } 110 111 while (fgets(buffer, sizeof(buffer), f)) 112 printf("%s", buffer); 113 114 fclose(f); 115 } 116 117 static int str_to_prot(char *prot_str) 118 { 119 int prot = 0; 120 121 if ((strchr(prot_str, 'r')) != NULL) 122 prot |= PROT_READ; 123 if ((strchr(prot_str, 'w')) != NULL) 124 prot |= PROT_WRITE; 125 if ((strchr(prot_str, 'x')) != NULL) 126 prot |= PROT_EXEC; 127 128 return prot; 129 } 130 131 static void print_help(void) 132 { 133 printf("usage: ksm_tests [-h] <test type> [-a prot] [-p page_count] [-l timeout]\n" 134 "[-z use_zero_pages] [-m merge_across_nodes] [-s size]\n"); 135 136 printf("Supported <test type>:\n" 137 " -M (page merging)\n" 138 " -Z (zero pages merging)\n" 139 " -N (merging of pages in different NUMA nodes)\n" 140 " -U (page unmerging)\n" 141 " -P evaluate merging time and speed.\n" 142 " For this test, the size of duplicated memory area (in MiB)\n" 143 " must be provided using -s option\n" 144 " -H evaluate merging time and speed of area allocated mostly with huge pages\n" 145 " For this test, the size of duplicated memory area (in MiB)\n" 146 " must be provided using -s option\n" 147 " -D evaluate unmerging time and speed when disabling KSM.\n" 148 " For this test, the size of duplicated memory area (in MiB)\n" 149 " must be provided using -s option\n" 150 " -C evaluate the time required to break COW of merged pages.\n\n"); 151 152 printf(" -a: specify the access protections of pages.\n" 153 " <prot> must be of the form [rwx].\n" 154 " Default: %s\n", KSM_PROT_STR_DEFAULT); 155 printf(" -p: specify the number of pages to test.\n" 156 " Default: %ld\n", KSM_PAGE_COUNT_DEFAULT); 157 printf(" -l: limit the maximum running time (in seconds) for a test.\n" 158 " Default: %d seconds\n", KSM_SCAN_LIMIT_SEC_DEFAULT); 159 printf(" -z: change use_zero_pages tunable\n" 160 " Default: %d\n", KSM_USE_ZERO_PAGES_DEFAULT); 161 printf(" -m: change merge_across_nodes tunable\n" 162 " Default: %d\n", KSM_MERGE_ACROSS_NODES_DEFAULT); 163 printf(" -d: turn debugging output on\n"); 164 printf(" -s: the size of duplicated memory area (in MiB)\n"); 165 printf(" -t: KSM merge type\n" 166 " Default: 0\n" 167 " 0: madvise merging\n" 168 " 1: prctl merging\n"); 169 170 exit(0); 171 } 172 173 static void *allocate_memory(void *ptr, int prot, int mapping, char data, size_t map_size) 174 { 175 void *map_ptr = mmap(ptr, map_size, PROT_WRITE, mapping, -1, 0); 176 177 if (map_ptr == MAP_FAILED) { 178 ksft_perror("mmap"); 179 return NULL; 180 } 181 memset(map_ptr, data, map_size); 182 if (mprotect(map_ptr, map_size, prot)) { 183 ksft_perror("mprotect"); 184 munmap(map_ptr, map_size); 185 return NULL; 186 } 187 188 return map_ptr; 189 } 190 191 static int ksm_do_scan(int scan_count, struct timespec start_time, int timeout) 192 { 193 struct timespec cur_time; 194 unsigned long cur_scan, init_scan; 195 196 if (ksm_read_sysfs(KSM_FP("full_scans"), &init_scan)) 197 return 1; 198 cur_scan = init_scan; 199 200 while (cur_scan < init_scan + scan_count) { 201 if (ksm_read_sysfs(KSM_FP("full_scans"), &cur_scan)) 202 return 1; 203 if (clock_gettime(CLOCK_MONOTONIC_RAW, &cur_time)) { 204 ksft_perror("clock_gettime"); 205 return 1; 206 } 207 if ((cur_time.tv_sec - start_time.tv_sec) > timeout) { 208 ksft_print_msg("Scan time limit exceeded\n"); 209 return 1; 210 } 211 } 212 213 return 0; 214 } 215 216 static int ksm_merge_pages(int merge_type, void *addr, size_t size, 217 struct timespec start_time, int timeout) 218 { 219 if (merge_type == KSM_MERGE_MADVISE) { 220 if (madvise(addr, size, MADV_MERGEABLE)) { 221 ksft_perror("madvise"); 222 return 1; 223 } 224 } else if (merge_type == KSM_MERGE_PRCTL) { 225 if (prctl(PR_SET_MEMORY_MERGE, 1, 0, 0, 0)) { 226 ksft_perror("prctl"); 227 return 1; 228 } 229 } 230 231 if (ksm_write_sysfs(KSM_FP("run"), 1)) 232 return 1; 233 234 /* Since merging occurs only after 2 scans, make sure to get at least 2 full scans */ 235 if (ksm_do_scan(2, start_time, timeout)) 236 return 1; 237 238 return 0; 239 } 240 241 static int ksm_unmerge_pages(void *addr, size_t size, 242 struct timespec start_time, int timeout) 243 { 244 if (madvise(addr, size, MADV_UNMERGEABLE)) { 245 ksft_perror("madvise"); 246 return 1; 247 } 248 return 0; 249 } 250 251 static bool assert_ksm_pages_count(long dupl_page_count) 252 { 253 unsigned long max_page_sharing, pages_sharing, pages_shared; 254 255 if (ksm_read_sysfs(KSM_FP("pages_shared"), &pages_shared) || 256 ksm_read_sysfs(KSM_FP("pages_sharing"), &pages_sharing) || 257 ksm_read_sysfs(KSM_FP("max_page_sharing"), &max_page_sharing)) 258 return false; 259 260 if (debug) { 261 ksm_print_sysfs(); 262 ksm_print_procfs(); 263 } 264 265 /* 266 * Since there must be at least 2 pages for merging and 1 page can be 267 * shared with the limited number of pages (max_page_sharing), sometimes 268 * there are 'leftover' pages that cannot be merged. For example, if there 269 * are 11 pages and max_page_sharing = 10, then only 10 pages will be 270 * merged and the 11th page won't be affected. As a result, when the number 271 * of duplicate pages is divided by max_page_sharing and the remainder is 1, 272 * pages_shared and pages_sharing values will be equal between dupl_page_count 273 * and dupl_page_count - 1. 274 */ 275 if (dupl_page_count % max_page_sharing == 1 || dupl_page_count % max_page_sharing == 0) { 276 if (pages_shared == dupl_page_count / max_page_sharing && 277 pages_sharing == pages_shared * (max_page_sharing - 1)) 278 return true; 279 } else { 280 if (pages_shared == (dupl_page_count / max_page_sharing + 1) && 281 pages_sharing == dupl_page_count - pages_shared) 282 return true; 283 } 284 285 return false; 286 } 287 288 static int ksm_save_def(struct ksm_sysfs *ksm_sysfs) 289 { 290 if (ksm_read_sysfs(KSM_FP("max_page_sharing"), &ksm_sysfs->max_page_sharing) || 291 numa_available() ? 0 : 292 ksm_read_sysfs(KSM_FP("merge_across_nodes"), &ksm_sysfs->merge_across_nodes) || 293 ksm_read_sysfs(KSM_FP("sleep_millisecs"), &ksm_sysfs->sleep_millisecs) || 294 ksm_read_sysfs(KSM_FP("pages_to_scan"), &ksm_sysfs->pages_to_scan) || 295 ksm_read_sysfs(KSM_FP("run"), &ksm_sysfs->run) || 296 ksm_read_sysfs(KSM_FP("stable_node_chains_prune_millisecs"), 297 &ksm_sysfs->stable_node_chains_prune_millisecs) || 298 ksm_read_sysfs(KSM_FP("use_zero_pages"), &ksm_sysfs->use_zero_pages)) 299 return 1; 300 301 return 0; 302 } 303 304 static int ksm_restore(struct ksm_sysfs *ksm_sysfs) 305 { 306 if (ksm_write_sysfs(KSM_FP("max_page_sharing"), ksm_sysfs->max_page_sharing) || 307 numa_available() ? 0 : 308 ksm_write_sysfs(KSM_FP("merge_across_nodes"), ksm_sysfs->merge_across_nodes) || 309 ksm_write_sysfs(KSM_FP("pages_to_scan"), ksm_sysfs->pages_to_scan) || 310 ksm_write_sysfs(KSM_FP("run"), ksm_sysfs->run) || 311 ksm_write_sysfs(KSM_FP("sleep_millisecs"), ksm_sysfs->sleep_millisecs) || 312 ksm_write_sysfs(KSM_FP("stable_node_chains_prune_millisecs"), 313 ksm_sysfs->stable_node_chains_prune_millisecs) || 314 ksm_write_sysfs(KSM_FP("use_zero_pages"), ksm_sysfs->use_zero_pages)) 315 return 1; 316 317 return 0; 318 } 319 320 static int check_ksm_merge(int merge_type, int mapping, int prot, 321 long page_count, int timeout, size_t page_size) 322 { 323 void *map_ptr; 324 struct timespec start_time; 325 326 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 327 ksft_perror("clock_gettime"); 328 return KSFT_FAIL; 329 } 330 331 /* fill pages with the same data and merge them */ 332 map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count); 333 if (!map_ptr) 334 return KSFT_FAIL; 335 336 if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout)) 337 goto err_out; 338 339 /* verify that the right number of pages are merged */ 340 if (assert_ksm_pages_count(page_count)) { 341 munmap(map_ptr, page_size * page_count); 342 if (merge_type == KSM_MERGE_PRCTL) 343 prctl(PR_SET_MEMORY_MERGE, 0, 0, 0, 0); 344 return KSFT_PASS; 345 } 346 347 err_out: 348 munmap(map_ptr, page_size * page_count); 349 return KSFT_FAIL; 350 } 351 352 static int check_ksm_unmerge(int merge_type, int mapping, int prot, int timeout, size_t page_size) 353 { 354 void *map_ptr; 355 struct timespec start_time; 356 int page_count = 2; 357 358 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 359 ksft_perror("clock_gettime"); 360 return KSFT_FAIL; 361 } 362 363 /* fill pages with the same data and merge them */ 364 map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count); 365 if (!map_ptr) 366 return KSFT_FAIL; 367 368 if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout)) 369 goto err_out; 370 371 /* change 1 byte in each of the 2 pages -- KSM must automatically unmerge them */ 372 memset(map_ptr, '-', 1); 373 memset(map_ptr + page_size, '+', 1); 374 375 /* get at least 1 scan, so KSM can detect that the pages were modified */ 376 if (ksm_do_scan(1, start_time, timeout)) 377 goto err_out; 378 379 /* check that unmerging was successful and 0 pages are currently merged */ 380 if (assert_ksm_pages_count(0)) { 381 munmap(map_ptr, page_size * page_count); 382 return KSFT_PASS; 383 } 384 385 err_out: 386 munmap(map_ptr, page_size * page_count); 387 return KSFT_FAIL; 388 } 389 390 static int check_ksm_zero_page_merge(int merge_type, int mapping, int prot, long page_count, 391 int timeout, bool use_zero_pages, size_t page_size) 392 { 393 void *map_ptr; 394 struct timespec start_time; 395 396 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 397 ksft_perror("clock_gettime"); 398 return KSFT_FAIL; 399 } 400 401 if (ksm_write_sysfs(KSM_FP("use_zero_pages"), use_zero_pages)) 402 return KSFT_FAIL; 403 404 /* fill pages with zero and try to merge them */ 405 map_ptr = allocate_memory(NULL, prot, mapping, 0, page_size * page_count); 406 if (!map_ptr) 407 return KSFT_FAIL; 408 409 if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout)) 410 goto err_out; 411 412 /* 413 * verify that the right number of pages are merged: 414 * 1) if use_zero_pages is set to 1, empty pages are merged 415 * with the kernel zero page instead of with each other; 416 * 2) if use_zero_pages is set to 0, empty pages are not treated specially 417 * and merged as usual. 418 */ 419 if (use_zero_pages && !assert_ksm_pages_count(0)) 420 goto err_out; 421 else if (!use_zero_pages && !assert_ksm_pages_count(page_count)) 422 goto err_out; 423 424 munmap(map_ptr, page_size * page_count); 425 return KSFT_PASS; 426 427 err_out: 428 munmap(map_ptr, page_size * page_count); 429 return KSFT_FAIL; 430 } 431 432 static int get_next_mem_node(int node) 433 { 434 435 long node_size; 436 int mem_node = 0; 437 int i, max_node = numa_max_node(); 438 439 for (i = node + 1; i <= max_node + node; i++) { 440 mem_node = i % (max_node + 1); 441 node_size = numa_node_size(mem_node, NULL); 442 if (node_size > 0) 443 break; 444 } 445 return mem_node; 446 } 447 448 static int get_first_mem_node(void) 449 { 450 return get_next_mem_node(numa_max_node()); 451 } 452 453 static int check_ksm_numa_merge(int merge_type, int mapping, int prot, int timeout, 454 bool merge_across_nodes, size_t page_size) 455 { 456 void *numa1_map_ptr, *numa2_map_ptr; 457 struct timespec start_time; 458 int page_count = 2; 459 int first_node; 460 461 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 462 ksft_perror("clock_gettime"); 463 return KSFT_FAIL; 464 } 465 466 if (numa_available() < 0) { 467 ksft_print_msg("NUMA support not enabled\n"); 468 return KSFT_SKIP; 469 } 470 if (numa_num_configured_nodes() <= 1) { 471 ksft_print_msg("At least 2 NUMA nodes must be available\n"); 472 return KSFT_SKIP; 473 } 474 if (ksm_write_sysfs(KSM_FP("merge_across_nodes"), merge_across_nodes)) 475 return KSFT_FAIL; 476 477 /* allocate 2 pages in 2 different NUMA nodes and fill them with the same data */ 478 first_node = get_first_mem_node(); 479 numa1_map_ptr = numa_alloc_onnode(page_size, first_node); 480 numa2_map_ptr = numa_alloc_onnode(page_size, get_next_mem_node(first_node)); 481 if (!numa1_map_ptr || !numa2_map_ptr) { 482 ksft_perror("numa_alloc_onnode"); 483 return KSFT_FAIL; 484 } 485 486 memset(numa1_map_ptr, '*', page_size); 487 memset(numa2_map_ptr, '*', page_size); 488 489 /* try to merge the pages */ 490 if (ksm_merge_pages(merge_type, numa1_map_ptr, page_size, start_time, timeout) || 491 ksm_merge_pages(merge_type, numa2_map_ptr, page_size, start_time, timeout)) 492 goto err_out; 493 494 /* 495 * verify that the right number of pages are merged: 496 * 1) if merge_across_nodes was enabled, 2 duplicate pages will be merged; 497 * 2) if merge_across_nodes = 0, there must be 0 merged pages, since there is 498 * only 1 unique page in each node and they can't be shared. 499 */ 500 if (merge_across_nodes && !assert_ksm_pages_count(page_count)) 501 goto err_out; 502 else if (!merge_across_nodes && !assert_ksm_pages_count(0)) 503 goto err_out; 504 505 numa_free(numa1_map_ptr, page_size); 506 numa_free(numa2_map_ptr, page_size); 507 return KSFT_PASS; 508 509 err_out: 510 numa_free(numa1_map_ptr, page_size); 511 numa_free(numa2_map_ptr, page_size); 512 return KSFT_FAIL; 513 } 514 515 static int ksm_merge_hugepages_time(int merge_type, int mapping, int prot, 516 int timeout, size_t map_size) 517 { 518 void *map_ptr, *map_ptr_orig; 519 struct timespec start_time, end_time; 520 unsigned long scan_time_ns; 521 int pagemap_fd, n_normal_pages, n_huge_pages; 522 523 if (!thp_is_enabled()) { 524 ksft_print_msg("Transparent Hugepages not available\n"); 525 return KSFT_SKIP; 526 } 527 528 map_size *= MB; 529 size_t len = map_size; 530 531 len -= len % HPAGE_SIZE; 532 map_ptr_orig = mmap(NULL, len + HPAGE_SIZE, PROT_READ | PROT_WRITE, 533 MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE, -1, 0); 534 map_ptr = map_ptr_orig + HPAGE_SIZE - (uintptr_t)map_ptr_orig % HPAGE_SIZE; 535 536 if (map_ptr_orig == MAP_FAILED) 537 err(2, "initial mmap"); 538 539 if (madvise(map_ptr, len, MADV_HUGEPAGE)) 540 err(2, "MADV_HUGEPAGE"); 541 542 pagemap_fd = open("/proc/self/pagemap", O_RDONLY); 543 if (pagemap_fd < 0) 544 err(2, "open pagemap"); 545 546 n_normal_pages = 0; 547 n_huge_pages = 0; 548 for (void *p = map_ptr; p < map_ptr + len; p += HPAGE_SIZE) { 549 if (allocate_transhuge(p, pagemap_fd) < 0) 550 n_normal_pages++; 551 else 552 n_huge_pages++; 553 } 554 ksft_print_msg("Number of normal pages: %d\n", n_normal_pages); 555 ksft_print_msg("Number of huge pages: %d\n", n_huge_pages); 556 557 memset(map_ptr, '*', len); 558 559 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 560 ksft_perror("clock_gettime"); 561 goto err_out; 562 } 563 if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout)) 564 goto err_out; 565 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { 566 ksft_perror("clock_gettime"); 567 goto err_out; 568 } 569 570 scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + 571 (end_time.tv_nsec - start_time.tv_nsec); 572 573 ksft_print_msg("Total size: %lu MiB\n", map_size / MB); 574 ksft_print_msg("Total time: %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC, 575 scan_time_ns % NSEC_PER_SEC); 576 ksft_print_msg("Average speed: %.3f MiB/s\n", (map_size / MB) / 577 ((double)scan_time_ns / NSEC_PER_SEC)); 578 579 munmap(map_ptr_orig, len + HPAGE_SIZE); 580 return KSFT_PASS; 581 582 err_out: 583 munmap(map_ptr_orig, len + HPAGE_SIZE); 584 return KSFT_FAIL; 585 } 586 587 static int ksm_merge_time(int merge_type, int mapping, int prot, int timeout, size_t map_size) 588 { 589 void *map_ptr; 590 struct timespec start_time, end_time; 591 unsigned long scan_time_ns; 592 593 map_size *= MB; 594 595 map_ptr = allocate_memory(NULL, prot, mapping, '*', map_size); 596 if (!map_ptr) 597 return KSFT_FAIL; 598 599 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 600 ksft_perror("clock_gettime"); 601 goto err_out; 602 } 603 if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout)) 604 goto err_out; 605 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { 606 ksft_perror("clock_gettime"); 607 goto err_out; 608 } 609 610 scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + 611 (end_time.tv_nsec - start_time.tv_nsec); 612 613 ksft_print_msg("Total size: %lu MiB\n", map_size / MB); 614 ksft_print_msg("Total time: %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC, 615 scan_time_ns % NSEC_PER_SEC); 616 ksft_print_msg("Average speed: %.3f MiB/s\n", (map_size / MB) / 617 ((double)scan_time_ns / NSEC_PER_SEC)); 618 619 munmap(map_ptr, map_size); 620 return KSFT_PASS; 621 622 err_out: 623 munmap(map_ptr, map_size); 624 return KSFT_FAIL; 625 } 626 627 static int ksm_unmerge_time(int merge_type, int mapping, int prot, int timeout, size_t map_size) 628 { 629 void *map_ptr; 630 struct timespec start_time, end_time; 631 unsigned long scan_time_ns; 632 633 map_size *= MB; 634 635 map_ptr = allocate_memory(NULL, prot, mapping, '*', map_size); 636 if (!map_ptr) 637 return KSFT_FAIL; 638 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 639 ksft_perror("clock_gettime"); 640 goto err_out; 641 } 642 if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout)) 643 goto err_out; 644 645 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 646 ksft_perror("clock_gettime"); 647 goto err_out; 648 } 649 if (ksm_unmerge_pages(map_ptr, map_size, start_time, timeout)) 650 goto err_out; 651 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { 652 ksft_perror("clock_gettime"); 653 goto err_out; 654 } 655 656 scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + 657 (end_time.tv_nsec - start_time.tv_nsec); 658 659 ksft_print_msg("Total size: %lu MiB\n", map_size / MB); 660 ksft_print_msg("Total time: %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC, 661 scan_time_ns % NSEC_PER_SEC); 662 ksft_print_msg("Average speed: %.3f MiB/s\n", (map_size / MB) / 663 ((double)scan_time_ns / NSEC_PER_SEC)); 664 665 munmap(map_ptr, map_size); 666 return KSFT_PASS; 667 668 err_out: 669 munmap(map_ptr, map_size); 670 return KSFT_FAIL; 671 } 672 673 static int ksm_cow_time(int merge_type, int mapping, int prot, int timeout, size_t page_size) 674 { 675 void *map_ptr; 676 struct timespec start_time, end_time; 677 unsigned long cow_time_ns; 678 679 /* page_count must be less than 2*page_size */ 680 size_t page_count = 4000; 681 682 map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count); 683 if (!map_ptr) 684 return KSFT_FAIL; 685 686 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 687 ksft_perror("clock_gettime"); 688 return KSFT_FAIL; 689 } 690 for (size_t i = 0; i < page_count - 1; i = i + 2) 691 memset(map_ptr + page_size * i, '-', 1); 692 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { 693 ksft_perror("clock_gettime"); 694 return KSFT_FAIL; 695 } 696 697 cow_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + 698 (end_time.tv_nsec - start_time.tv_nsec); 699 700 ksft_print_msg("Total size: %lu MiB\n\n", (page_size * page_count) / MB); 701 ksft_print_msg("Not merged pages:\n"); 702 ksft_print_msg("Total time: %ld.%09ld s\n", cow_time_ns / NSEC_PER_SEC, 703 cow_time_ns % NSEC_PER_SEC); 704 ksft_print_msg("Average speed: %.3f MiB/s\n\n", ((page_size * (page_count / 2)) / MB) / 705 ((double)cow_time_ns / NSEC_PER_SEC)); 706 707 /* Create 2000 pairs of duplicate pages */ 708 for (size_t i = 0; i < page_count - 1; i = i + 2) { 709 memset(map_ptr + page_size * i, '+', i / 2 + 1); 710 memset(map_ptr + page_size * (i + 1), '+', i / 2 + 1); 711 } 712 if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout)) 713 goto err_out; 714 715 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 716 ksft_perror("clock_gettime"); 717 goto err_out; 718 } 719 for (size_t i = 0; i < page_count - 1; i = i + 2) 720 memset(map_ptr + page_size * i, '-', 1); 721 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { 722 ksft_perror("clock_gettime"); 723 goto err_out; 724 } 725 726 cow_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + 727 (end_time.tv_nsec - start_time.tv_nsec); 728 729 ksft_print_msg("Merged pages:\n"); 730 ksft_print_msg("Total time: %ld.%09ld s\n", cow_time_ns / NSEC_PER_SEC, 731 cow_time_ns % NSEC_PER_SEC); 732 ksft_print_msg("Average speed: %.3f MiB/s\n", ((page_size * (page_count / 2)) / MB) / 733 ((double)cow_time_ns / NSEC_PER_SEC)); 734 735 munmap(map_ptr, page_size * page_count); 736 return KSFT_PASS; 737 738 err_out: 739 munmap(map_ptr, page_size * page_count); 740 return KSFT_FAIL; 741 } 742 743 int main(int argc, char *argv[]) 744 { 745 int ret = 0, opt; 746 int prot = 0; 747 int ksm_scan_limit_sec = KSM_SCAN_LIMIT_SEC_DEFAULT; 748 int merge_type = KSM_MERGE_TYPE_DEFAULT; 749 long page_count = KSM_PAGE_COUNT_DEFAULT; 750 size_t page_size = sysconf(_SC_PAGESIZE); 751 struct ksm_sysfs ksm_sysfs_old; 752 int test_name = CHECK_KSM_MERGE; 753 bool use_zero_pages = KSM_USE_ZERO_PAGES_DEFAULT; 754 bool merge_across_nodes = KSM_MERGE_ACROSS_NODES_DEFAULT; 755 long size_MB = 0; 756 const char *test_descr = "KSM merging"; 757 758 ksft_print_header(); 759 ksft_set_plan(1); 760 761 while ((opt = getopt(argc, argv, "dha:p:l:z:m:s:t:MUZNPCHD")) != -1) { 762 switch (opt) { 763 case 'a': 764 prot = str_to_prot(optarg); 765 break; 766 case 'p': 767 page_count = atol(optarg); 768 if (page_count <= 0) 769 ksft_exit_fail_msg("The number of pages must be greater than 0\n"); 770 break; 771 case 'l': 772 ksm_scan_limit_sec = atoi(optarg); 773 if (ksm_scan_limit_sec <= 0) 774 ksft_exit_fail_msg("Timeout value must be greater than 0\n"); 775 break; 776 case 'h': 777 print_help(); 778 break; 779 case 'z': 780 if (strcmp(optarg, "0") == 0) 781 use_zero_pages = 0; 782 else 783 use_zero_pages = 1; 784 break; 785 case 'm': 786 if (strcmp(optarg, "0") == 0) 787 merge_across_nodes = 0; 788 else 789 merge_across_nodes = 1; 790 break; 791 case 'd': 792 debug = 1; 793 break; 794 case 's': 795 size_MB = atoi(optarg); 796 if (size_MB <= 0) 797 ksft_exit_fail_msg("Size must be greater than 0\n"); 798 break; 799 case 't': 800 { 801 int tmp = atoi(optarg); 802 803 if (tmp < 0 || tmp > KSM_MERGE_LAST) 804 ksft_exit_fail_msg("Invalid merge type\n"); 805 merge_type = tmp; 806 } 807 break; 808 case 'M': 809 break; 810 case 'U': 811 test_name = CHECK_KSM_UNMERGE; 812 break; 813 case 'Z': 814 test_name = CHECK_KSM_ZERO_PAGE_MERGE; 815 break; 816 case 'N': 817 test_name = CHECK_KSM_NUMA_MERGE; 818 break; 819 case 'P': 820 test_name = KSM_MERGE_TIME; 821 break; 822 case 'H': 823 test_name = KSM_MERGE_TIME_HUGE_PAGES; 824 break; 825 case 'D': 826 test_name = KSM_UNMERGE_TIME; 827 break; 828 case 'C': 829 test_name = KSM_COW_TIME; 830 break; 831 default: 832 ksft_exit_fail_msg("Unknown option\n"); 833 } 834 } 835 836 if (prot == 0) 837 prot = str_to_prot(KSM_PROT_STR_DEFAULT); 838 839 if (access(KSM_SYSFS_PATH, F_OK)) 840 ksft_exit_skip("Config KSM not enabled\n"); 841 842 if (ksm_save_def(&ksm_sysfs_old)) 843 ksft_exit_fail_msg("Cannot save default tunables\n"); 844 845 if (ksm_write_sysfs(KSM_FP("run"), 2) || 846 ksm_write_sysfs(KSM_FP("sleep_millisecs"), 0) || 847 numa_available() ? 0 : 848 ksm_write_sysfs(KSM_FP("merge_across_nodes"), 1) || 849 ksm_write_sysfs(KSM_FP("pages_to_scan"), page_count)) 850 ksft_exit_fail_msg("Cannot set up KSM tunables\n"); 851 852 switch (test_name) { 853 case CHECK_KSM_MERGE: 854 test_descr = "KSM merging"; 855 ret = check_ksm_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, page_count, 856 ksm_scan_limit_sec, page_size); 857 break; 858 case CHECK_KSM_UNMERGE: 859 test_descr = "KSM unmerging"; 860 ret = check_ksm_unmerge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, 861 ksm_scan_limit_sec, page_size); 862 break; 863 case CHECK_KSM_ZERO_PAGE_MERGE: 864 test_descr = "KSM zero page merging"; 865 ret = check_ksm_zero_page_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, 866 page_count, ksm_scan_limit_sec, use_zero_pages, 867 page_size); 868 break; 869 case CHECK_KSM_NUMA_MERGE: 870 test_descr = "KSM NUMA merging"; 871 ret = check_ksm_numa_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, 872 ksm_scan_limit_sec, merge_across_nodes, page_size); 873 break; 874 case KSM_MERGE_TIME: 875 if (size_MB == 0) 876 ksft_exit_fail_msg("Option '-s' is required\n"); 877 test_descr = "KSM merge time"; 878 ret = ksm_merge_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, 879 ksm_scan_limit_sec, size_MB); 880 break; 881 case KSM_MERGE_TIME_HUGE_PAGES: 882 if (size_MB == 0) 883 ksft_exit_fail_msg("Option '-s' is required\n"); 884 test_descr = "KSM merge time with huge pages"; 885 ret = ksm_merge_hugepages_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, 886 ksm_scan_limit_sec, size_MB); 887 break; 888 case KSM_UNMERGE_TIME: 889 if (size_MB == 0) 890 ksft_exit_fail_msg("Option '-s' is required\n"); 891 test_descr = "KSM unmerge time"; 892 ret = ksm_unmerge_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, 893 ksm_scan_limit_sec, size_MB); 894 break; 895 case KSM_COW_TIME: 896 test_descr = "KSM COW time"; 897 ret = ksm_cow_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, 898 ksm_scan_limit_sec, page_size); 899 break; 900 } 901 902 if (ksm_restore(&ksm_sysfs_old)) 903 ksft_print_msg("Cannot restore default tunables\n"); 904 905 ksft_test_result_report(ret, "%s\n", test_descr); 906 907 ksft_finished(); 908 } 909