1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <sys/mman.h> 4 #include <sys/prctl.h> 5 #include <sys/wait.h> 6 #include <stdbool.h> 7 #include <time.h> 8 #include <string.h> 9 #include <numa.h> 10 #include <unistd.h> 11 #include <fcntl.h> 12 #include <stdint.h> 13 #include <err.h> 14 15 #include "kselftest.h" 16 #include <include/vdso/time64.h> 17 #include "vm_util.h" 18 19 #define KSM_SYSFS_PATH "/sys/kernel/mm/ksm/" 20 #define KSM_FP(s) (KSM_SYSFS_PATH s) 21 #define KSM_SCAN_LIMIT_SEC_DEFAULT 120 22 #define KSM_PAGE_COUNT_DEFAULT 10l 23 #define KSM_PROT_STR_DEFAULT "rw" 24 #define KSM_USE_ZERO_PAGES_DEFAULT false 25 #define KSM_MERGE_ACROSS_NODES_DEFAULT true 26 #define KSM_MERGE_TYPE_DEFAULT 0 27 #define MB (1ul << 20) 28 29 struct ksm_sysfs { 30 unsigned long max_page_sharing; 31 unsigned long merge_across_nodes; 32 unsigned long pages_to_scan; 33 unsigned long run; 34 unsigned long sleep_millisecs; 35 unsigned long stable_node_chains_prune_millisecs; 36 unsigned long use_zero_pages; 37 }; 38 39 enum ksm_merge_type { 40 KSM_MERGE_MADVISE, 41 KSM_MERGE_PRCTL, 42 KSM_MERGE_LAST = KSM_MERGE_PRCTL 43 }; 44 45 enum ksm_test_name { 46 CHECK_KSM_MERGE, 47 CHECK_KSM_UNMERGE, 48 CHECK_KSM_GET_MERGE_TYPE, 49 CHECK_KSM_ZERO_PAGE_MERGE, 50 CHECK_KSM_NUMA_MERGE, 51 KSM_MERGE_TIME, 52 KSM_MERGE_TIME_HUGE_PAGES, 53 KSM_UNMERGE_TIME, 54 KSM_COW_TIME 55 }; 56 57 int debug; 58 59 static int ksm_write_sysfs(const char *file_path, unsigned long val) 60 { 61 return write_sysfs(file_path, val); 62 } 63 64 static int ksm_read_sysfs(const char *file_path, unsigned long *val) 65 { 66 return read_sysfs(file_path, val); 67 } 68 69 static void ksm_print_sysfs(void) 70 { 71 unsigned long max_page_sharing, pages_sharing, pages_shared; 72 unsigned long full_scans, pages_unshared, pages_volatile; 73 unsigned long stable_node_chains, stable_node_dups; 74 long general_profit; 75 76 if (ksm_read_sysfs(KSM_FP("pages_shared"), &pages_shared) || 77 ksm_read_sysfs(KSM_FP("pages_sharing"), &pages_sharing) || 78 ksm_read_sysfs(KSM_FP("max_page_sharing"), &max_page_sharing) || 79 ksm_read_sysfs(KSM_FP("full_scans"), &full_scans) || 80 ksm_read_sysfs(KSM_FP("pages_unshared"), &pages_unshared) || 81 ksm_read_sysfs(KSM_FP("pages_volatile"), &pages_volatile) || 82 ksm_read_sysfs(KSM_FP("stable_node_chains"), &stable_node_chains) || 83 ksm_read_sysfs(KSM_FP("stable_node_dups"), &stable_node_dups) || 84 ksm_read_sysfs(KSM_FP("general_profit"), (unsigned long *)&general_profit)) 85 return; 86 87 printf("pages_shared : %lu\n", pages_shared); 88 printf("pages_sharing : %lu\n", pages_sharing); 89 printf("max_page_sharing : %lu\n", max_page_sharing); 90 printf("full_scans : %lu\n", full_scans); 91 printf("pages_unshared : %lu\n", pages_unshared); 92 printf("pages_volatile : %lu\n", pages_volatile); 93 printf("stable_node_chains: %lu\n", stable_node_chains); 94 printf("stable_node_dups : %lu\n", stable_node_dups); 95 printf("general_profit : %ld\n", general_profit); 96 } 97 98 static void ksm_print_procfs(void) 99 { 100 const char *file_name = "/proc/self/ksm_stat"; 101 char buffer[512]; 102 FILE *f = fopen(file_name, "r"); 103 104 if (!f) { 105 fprintf(stderr, "f %s\n", file_name); 106 perror("fopen"); 107 return; 108 } 109 110 while (fgets(buffer, sizeof(buffer), f)) 111 printf("%s", buffer); 112 113 fclose(f); 114 } 115 116 static int str_to_prot(char *prot_str) 117 { 118 int prot = 0; 119 120 if ((strchr(prot_str, 'r')) != NULL) 121 prot |= PROT_READ; 122 if ((strchr(prot_str, 'w')) != NULL) 123 prot |= PROT_WRITE; 124 if ((strchr(prot_str, 'x')) != NULL) 125 prot |= PROT_EXEC; 126 127 return prot; 128 } 129 130 static void print_help(void) 131 { 132 printf("usage: ksm_tests [-h] <test type> [-a prot] [-p page_count] [-l timeout]\n" 133 "[-z use_zero_pages] [-m merge_across_nodes] [-s size]\n"); 134 135 printf("Supported <test type>:\n" 136 " -M (page merging)\n" 137 " -Z (zero pages merging)\n" 138 " -N (merging of pages in different NUMA nodes)\n" 139 " -U (page unmerging)\n" 140 " -P evaluate merging time and speed.\n" 141 " For this test, the size of duplicated memory area (in MiB)\n" 142 " must be provided using -s option\n" 143 " -H evaluate merging time and speed of area allocated mostly with huge pages\n" 144 " For this test, the size of duplicated memory area (in MiB)\n" 145 " must be provided using -s option\n" 146 " -D evaluate unmerging time and speed when disabling KSM.\n" 147 " For this test, the size of duplicated memory area (in MiB)\n" 148 " must be provided using -s option\n" 149 " -C evaluate the time required to break COW of merged pages.\n\n"); 150 151 printf(" -a: specify the access protections of pages.\n" 152 " <prot> must be of the form [rwx].\n" 153 " Default: %s\n", KSM_PROT_STR_DEFAULT); 154 printf(" -p: specify the number of pages to test.\n" 155 " Default: %ld\n", KSM_PAGE_COUNT_DEFAULT); 156 printf(" -l: limit the maximum running time (in seconds) for a test.\n" 157 " Default: %d seconds\n", KSM_SCAN_LIMIT_SEC_DEFAULT); 158 printf(" -z: change use_zero_pages tunable\n" 159 " Default: %d\n", KSM_USE_ZERO_PAGES_DEFAULT); 160 printf(" -m: change merge_across_nodes tunable\n" 161 " Default: %d\n", KSM_MERGE_ACROSS_NODES_DEFAULT); 162 printf(" -d: turn debugging output on\n"); 163 printf(" -s: the size of duplicated memory area (in MiB)\n"); 164 printf(" -t: KSM merge type\n" 165 " Default: 0\n" 166 " 0: madvise merging\n" 167 " 1: prctl merging\n"); 168 169 exit(0); 170 } 171 172 static void *allocate_memory(void *ptr, int prot, int mapping, char data, size_t map_size) 173 { 174 void *map_ptr = mmap(ptr, map_size, PROT_WRITE, mapping, -1, 0); 175 176 if (map_ptr == MAP_FAILED) { 177 ksft_perror("mmap"); 178 return NULL; 179 } 180 memset(map_ptr, data, map_size); 181 if (mprotect(map_ptr, map_size, prot)) { 182 ksft_perror("mprotect"); 183 munmap(map_ptr, map_size); 184 return NULL; 185 } 186 187 return map_ptr; 188 } 189 190 static int ksm_do_scan(int scan_count, struct timespec start_time, int timeout) 191 { 192 struct timespec cur_time; 193 unsigned long cur_scan, init_scan; 194 195 if (ksm_read_sysfs(KSM_FP("full_scans"), &init_scan)) 196 return 1; 197 cur_scan = init_scan; 198 199 while (cur_scan < init_scan + scan_count) { 200 if (ksm_read_sysfs(KSM_FP("full_scans"), &cur_scan)) 201 return 1; 202 if (clock_gettime(CLOCK_MONOTONIC_RAW, &cur_time)) { 203 ksft_perror("clock_gettime"); 204 return 1; 205 } 206 if ((cur_time.tv_sec - start_time.tv_sec) > timeout) { 207 ksft_print_msg("Scan time limit exceeded\n"); 208 return 1; 209 } 210 } 211 212 return 0; 213 } 214 215 static int ksm_merge_pages(int merge_type, void *addr, size_t size, 216 struct timespec start_time, int timeout) 217 { 218 if (merge_type == KSM_MERGE_MADVISE) { 219 if (madvise(addr, size, MADV_MERGEABLE)) { 220 ksft_perror("madvise"); 221 return 1; 222 } 223 } else if (merge_type == KSM_MERGE_PRCTL) { 224 if (prctl(PR_SET_MEMORY_MERGE, 1, 0, 0, 0)) { 225 ksft_perror("prctl"); 226 return 1; 227 } 228 } 229 230 if (ksm_write_sysfs(KSM_FP("run"), 1)) 231 return 1; 232 233 /* Since merging occurs only after 2 scans, make sure to get at least 2 full scans */ 234 if (ksm_do_scan(2, start_time, timeout)) 235 return 1; 236 237 return 0; 238 } 239 240 static int ksm_unmerge_pages(void *addr, size_t size, 241 struct timespec start_time, int timeout) 242 { 243 if (madvise(addr, size, MADV_UNMERGEABLE)) { 244 ksft_perror("madvise"); 245 return 1; 246 } 247 return 0; 248 } 249 250 static bool assert_ksm_pages_count(long dupl_page_count) 251 { 252 unsigned long max_page_sharing, pages_sharing, pages_shared; 253 254 if (ksm_read_sysfs(KSM_FP("pages_shared"), &pages_shared) || 255 ksm_read_sysfs(KSM_FP("pages_sharing"), &pages_sharing) || 256 ksm_read_sysfs(KSM_FP("max_page_sharing"), &max_page_sharing)) 257 return false; 258 259 if (debug) { 260 ksm_print_sysfs(); 261 ksm_print_procfs(); 262 } 263 264 /* 265 * Since there must be at least 2 pages for merging and 1 page can be 266 * shared with the limited number of pages (max_page_sharing), sometimes 267 * there are 'leftover' pages that cannot be merged. For example, if there 268 * are 11 pages and max_page_sharing = 10, then only 10 pages will be 269 * merged and the 11th page won't be affected. As a result, when the number 270 * of duplicate pages is divided by max_page_sharing and the remainder is 1, 271 * pages_shared and pages_sharing values will be equal between dupl_page_count 272 * and dupl_page_count - 1. 273 */ 274 if (dupl_page_count % max_page_sharing == 1 || dupl_page_count % max_page_sharing == 0) { 275 if (pages_shared == dupl_page_count / max_page_sharing && 276 pages_sharing == pages_shared * (max_page_sharing - 1)) 277 return true; 278 } else { 279 if (pages_shared == (dupl_page_count / max_page_sharing + 1) && 280 pages_sharing == dupl_page_count - pages_shared) 281 return true; 282 } 283 284 return false; 285 } 286 287 static int ksm_save_def(struct ksm_sysfs *ksm_sysfs) 288 { 289 if (ksm_read_sysfs(KSM_FP("max_page_sharing"), &ksm_sysfs->max_page_sharing) || 290 (numa_available() ? 0 : 291 ksm_read_sysfs(KSM_FP("merge_across_nodes"), &ksm_sysfs->merge_across_nodes)) || 292 ksm_read_sysfs(KSM_FP("sleep_millisecs"), &ksm_sysfs->sleep_millisecs) || 293 ksm_read_sysfs(KSM_FP("pages_to_scan"), &ksm_sysfs->pages_to_scan) || 294 ksm_read_sysfs(KSM_FP("run"), &ksm_sysfs->run) || 295 ksm_read_sysfs(KSM_FP("stable_node_chains_prune_millisecs"), 296 &ksm_sysfs->stable_node_chains_prune_millisecs) || 297 ksm_read_sysfs(KSM_FP("use_zero_pages"), &ksm_sysfs->use_zero_pages)) 298 return 1; 299 300 return 0; 301 } 302 303 static int ksm_restore(struct ksm_sysfs *ksm_sysfs) 304 { 305 if (ksm_write_sysfs(KSM_FP("max_page_sharing"), ksm_sysfs->max_page_sharing) || 306 (numa_available() ? 0 : 307 ksm_write_sysfs(KSM_FP("merge_across_nodes"), ksm_sysfs->merge_across_nodes)) || 308 ksm_write_sysfs(KSM_FP("pages_to_scan"), ksm_sysfs->pages_to_scan) || 309 ksm_write_sysfs(KSM_FP("run"), ksm_sysfs->run) || 310 ksm_write_sysfs(KSM_FP("sleep_millisecs"), ksm_sysfs->sleep_millisecs) || 311 ksm_write_sysfs(KSM_FP("stable_node_chains_prune_millisecs"), 312 ksm_sysfs->stable_node_chains_prune_millisecs) || 313 ksm_write_sysfs(KSM_FP("use_zero_pages"), ksm_sysfs->use_zero_pages)) 314 return 1; 315 316 return 0; 317 } 318 319 static int check_ksm_merge(int merge_type, int mapping, int prot, 320 long page_count, int timeout, size_t page_size) 321 { 322 void *map_ptr; 323 struct timespec start_time; 324 325 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 326 ksft_perror("clock_gettime"); 327 return KSFT_FAIL; 328 } 329 330 /* fill pages with the same data and merge them */ 331 map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count); 332 if (!map_ptr) 333 return KSFT_FAIL; 334 335 if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout)) 336 goto err_out; 337 338 /* verify that the right number of pages are merged */ 339 if (assert_ksm_pages_count(page_count)) { 340 munmap(map_ptr, page_size * page_count); 341 if (merge_type == KSM_MERGE_PRCTL) 342 prctl(PR_SET_MEMORY_MERGE, 0, 0, 0, 0); 343 return KSFT_PASS; 344 } 345 346 err_out: 347 munmap(map_ptr, page_size * page_count); 348 return KSFT_FAIL; 349 } 350 351 static int check_ksm_unmerge(int merge_type, int mapping, int prot, int timeout, size_t page_size) 352 { 353 void *map_ptr; 354 struct timespec start_time; 355 int page_count = 2; 356 357 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 358 ksft_perror("clock_gettime"); 359 return KSFT_FAIL; 360 } 361 362 /* fill pages with the same data and merge them */ 363 map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count); 364 if (!map_ptr) 365 return KSFT_FAIL; 366 367 if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout)) 368 goto err_out; 369 370 /* change 1 byte in each of the 2 pages -- KSM must automatically unmerge them */ 371 memset(map_ptr, '-', 1); 372 memset(map_ptr + page_size, '+', 1); 373 374 /* get at least 1 scan, so KSM can detect that the pages were modified */ 375 if (ksm_do_scan(1, start_time, timeout)) 376 goto err_out; 377 378 /* check that unmerging was successful and 0 pages are currently merged */ 379 if (assert_ksm_pages_count(0)) { 380 munmap(map_ptr, page_size * page_count); 381 return KSFT_PASS; 382 } 383 384 err_out: 385 munmap(map_ptr, page_size * page_count); 386 return KSFT_FAIL; 387 } 388 389 static int check_ksm_zero_page_merge(int merge_type, int mapping, int prot, long page_count, 390 int timeout, bool use_zero_pages, size_t page_size) 391 { 392 void *map_ptr; 393 struct timespec start_time; 394 395 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 396 ksft_perror("clock_gettime"); 397 return KSFT_FAIL; 398 } 399 400 if (ksm_write_sysfs(KSM_FP("use_zero_pages"), use_zero_pages)) 401 return KSFT_FAIL; 402 403 /* fill pages with zero and try to merge them */ 404 map_ptr = allocate_memory(NULL, prot, mapping, 0, page_size * page_count); 405 if (!map_ptr) 406 return KSFT_FAIL; 407 408 if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout)) 409 goto err_out; 410 411 /* 412 * verify that the right number of pages are merged: 413 * 1) if use_zero_pages is set to 1, empty pages are merged 414 * with the kernel zero page instead of with each other; 415 * 2) if use_zero_pages is set to 0, empty pages are not treated specially 416 * and merged as usual. 417 */ 418 if (use_zero_pages && !assert_ksm_pages_count(0)) 419 goto err_out; 420 else if (!use_zero_pages && !assert_ksm_pages_count(page_count)) 421 goto err_out; 422 423 munmap(map_ptr, page_size * page_count); 424 return KSFT_PASS; 425 426 err_out: 427 munmap(map_ptr, page_size * page_count); 428 return KSFT_FAIL; 429 } 430 431 static int get_next_mem_node(int node) 432 { 433 434 long node_size; 435 int mem_node = 0; 436 int i, max_node = numa_max_node(); 437 438 for (i = node + 1; i <= max_node + node; i++) { 439 mem_node = i % (max_node + 1); 440 node_size = numa_node_size(mem_node, NULL); 441 if (node_size > 0) 442 return mem_node; 443 } 444 return -ENODEV; 445 } 446 447 static int get_first_mem_node(void) 448 { 449 return get_next_mem_node(numa_max_node()); 450 } 451 452 static int check_ksm_numa_merge(int merge_type, int mapping, int prot, int timeout, 453 bool merge_across_nodes, size_t page_size) 454 { 455 void *numa1_map_ptr, *numa2_map_ptr; 456 struct timespec start_time; 457 int first_node, second_node; 458 int page_count = 2; 459 460 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 461 ksft_perror("clock_gettime"); 462 return KSFT_FAIL; 463 } 464 465 if (numa_available() < 0) { 466 ksft_print_msg("NUMA support not enabled\n"); 467 return KSFT_SKIP; 468 } 469 first_node = get_first_mem_node(); 470 second_node = get_next_mem_node(first_node); 471 472 if (second_node < 0) { 473 ksft_print_msg("At least 2 NUMA nodes with memory must be available\n"); 474 return KSFT_SKIP; 475 } 476 if (ksm_write_sysfs(KSM_FP("merge_across_nodes"), merge_across_nodes)) 477 return KSFT_FAIL; 478 479 /* allocate 2 pages in 2 different NUMA nodes and fill them with the same data */ 480 numa1_map_ptr = numa_alloc_onnode(page_size, first_node); 481 numa2_map_ptr = numa_alloc_onnode(page_size, second_node); 482 if (!numa1_map_ptr || !numa2_map_ptr) { 483 ksft_perror("numa_alloc_onnode"); 484 return KSFT_FAIL; 485 } 486 487 memset(numa1_map_ptr, '*', page_size); 488 memset(numa2_map_ptr, '*', page_size); 489 490 /* try to merge the pages */ 491 if (ksm_merge_pages(merge_type, numa1_map_ptr, page_size, start_time, timeout) || 492 ksm_merge_pages(merge_type, numa2_map_ptr, page_size, start_time, timeout)) 493 goto err_out; 494 495 /* 496 * verify that the right number of pages are merged: 497 * 1) if merge_across_nodes was enabled, 2 duplicate pages will be merged; 498 * 2) if merge_across_nodes = 0, there must be 0 merged pages, since there is 499 * only 1 unique page in each node and they can't be shared. 500 */ 501 if (merge_across_nodes && !assert_ksm_pages_count(page_count)) 502 goto err_out; 503 else if (!merge_across_nodes && !assert_ksm_pages_count(0)) 504 goto err_out; 505 506 numa_free(numa1_map_ptr, page_size); 507 numa_free(numa2_map_ptr, page_size); 508 return KSFT_PASS; 509 510 err_out: 511 numa_free(numa1_map_ptr, page_size); 512 numa_free(numa2_map_ptr, page_size); 513 return KSFT_FAIL; 514 } 515 516 static int ksm_merge_hugepages_time(int merge_type, int mapping, int prot, 517 int timeout, size_t map_size) 518 { 519 void *map_ptr, *map_ptr_orig; 520 struct timespec start_time, end_time; 521 unsigned long scan_time_ns; 522 int pagemap_fd, n_normal_pages, n_huge_pages; 523 524 if (!thp_is_enabled()) { 525 ksft_print_msg("Transparent Hugepages not available\n"); 526 return KSFT_SKIP; 527 } 528 529 map_size *= MB; 530 size_t len = map_size; 531 532 len -= len % HPAGE_SIZE; 533 map_ptr_orig = mmap(NULL, len + HPAGE_SIZE, PROT_READ | PROT_WRITE, 534 MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE, -1, 0); 535 map_ptr = map_ptr_orig + HPAGE_SIZE - (uintptr_t)map_ptr_orig % HPAGE_SIZE; 536 537 if (map_ptr_orig == MAP_FAILED) 538 err(2, "initial mmap"); 539 540 if (madvise(map_ptr, len, MADV_HUGEPAGE)) 541 err(2, "MADV_HUGEPAGE"); 542 543 pagemap_fd = open("/proc/self/pagemap", O_RDONLY); 544 if (pagemap_fd < 0) 545 err(2, "open pagemap"); 546 547 n_normal_pages = 0; 548 n_huge_pages = 0; 549 for (void *p = map_ptr; p < map_ptr + len; p += HPAGE_SIZE) { 550 if (allocate_transhuge(p, pagemap_fd) < 0) 551 n_normal_pages++; 552 else 553 n_huge_pages++; 554 } 555 ksft_print_msg("Number of normal pages: %d\n", n_normal_pages); 556 ksft_print_msg("Number of huge pages: %d\n", n_huge_pages); 557 558 memset(map_ptr, '*', len); 559 560 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 561 ksft_perror("clock_gettime"); 562 goto err_out; 563 } 564 if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout)) 565 goto err_out; 566 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { 567 ksft_perror("clock_gettime"); 568 goto err_out; 569 } 570 571 scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + 572 (end_time.tv_nsec - start_time.tv_nsec); 573 574 ksft_print_msg("Total size: %lu MiB\n", map_size / MB); 575 ksft_print_msg("Total time: %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC, 576 scan_time_ns % NSEC_PER_SEC); 577 ksft_print_msg("Average speed: %.3f MiB/s\n", (map_size / MB) / 578 ((double)scan_time_ns / NSEC_PER_SEC)); 579 580 munmap(map_ptr_orig, len + HPAGE_SIZE); 581 return KSFT_PASS; 582 583 err_out: 584 munmap(map_ptr_orig, len + HPAGE_SIZE); 585 return KSFT_FAIL; 586 } 587 588 static int ksm_merge_time(int merge_type, int mapping, int prot, int timeout, size_t map_size) 589 { 590 void *map_ptr; 591 struct timespec start_time, end_time; 592 unsigned long scan_time_ns; 593 594 map_size *= MB; 595 596 map_ptr = allocate_memory(NULL, prot, mapping, '*', map_size); 597 if (!map_ptr) 598 return KSFT_FAIL; 599 600 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 601 ksft_perror("clock_gettime"); 602 goto err_out; 603 } 604 if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout)) 605 goto err_out; 606 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { 607 ksft_perror("clock_gettime"); 608 goto err_out; 609 } 610 611 scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + 612 (end_time.tv_nsec - start_time.tv_nsec); 613 614 ksft_print_msg("Total size: %lu MiB\n", map_size / MB); 615 ksft_print_msg("Total time: %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC, 616 scan_time_ns % NSEC_PER_SEC); 617 ksft_print_msg("Average speed: %.3f MiB/s\n", (map_size / MB) / 618 ((double)scan_time_ns / NSEC_PER_SEC)); 619 620 munmap(map_ptr, map_size); 621 return KSFT_PASS; 622 623 err_out: 624 munmap(map_ptr, map_size); 625 return KSFT_FAIL; 626 } 627 628 static int ksm_unmerge_time(int merge_type, int mapping, int prot, int timeout, size_t map_size) 629 { 630 void *map_ptr; 631 struct timespec start_time, end_time; 632 unsigned long scan_time_ns; 633 634 map_size *= MB; 635 636 map_ptr = allocate_memory(NULL, prot, mapping, '*', map_size); 637 if (!map_ptr) 638 return KSFT_FAIL; 639 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 640 ksft_perror("clock_gettime"); 641 goto err_out; 642 } 643 if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout)) 644 goto err_out; 645 646 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 647 ksft_perror("clock_gettime"); 648 goto err_out; 649 } 650 if (ksm_unmerge_pages(map_ptr, map_size, start_time, timeout)) 651 goto err_out; 652 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { 653 ksft_perror("clock_gettime"); 654 goto err_out; 655 } 656 657 scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + 658 (end_time.tv_nsec - start_time.tv_nsec); 659 660 ksft_print_msg("Total size: %lu MiB\n", map_size / MB); 661 ksft_print_msg("Total time: %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC, 662 scan_time_ns % NSEC_PER_SEC); 663 ksft_print_msg("Average speed: %.3f MiB/s\n", (map_size / MB) / 664 ((double)scan_time_ns / NSEC_PER_SEC)); 665 666 munmap(map_ptr, map_size); 667 return KSFT_PASS; 668 669 err_out: 670 munmap(map_ptr, map_size); 671 return KSFT_FAIL; 672 } 673 674 static int ksm_cow_time(int merge_type, int mapping, int prot, int timeout, size_t page_size) 675 { 676 void *map_ptr; 677 struct timespec start_time, end_time; 678 unsigned long cow_time_ns; 679 680 /* page_count must be less than 2*page_size */ 681 size_t page_count = 4000; 682 683 map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count); 684 if (!map_ptr) 685 return KSFT_FAIL; 686 687 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 688 ksft_perror("clock_gettime"); 689 return KSFT_FAIL; 690 } 691 for (size_t i = 0; i < page_count - 1; i = i + 2) 692 memset(map_ptr + page_size * i, '-', 1); 693 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { 694 ksft_perror("clock_gettime"); 695 return KSFT_FAIL; 696 } 697 698 cow_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + 699 (end_time.tv_nsec - start_time.tv_nsec); 700 701 ksft_print_msg("Total size: %lu MiB\n\n", (page_size * page_count) / MB); 702 ksft_print_msg("Not merged pages:\n"); 703 ksft_print_msg("Total time: %ld.%09ld s\n", cow_time_ns / NSEC_PER_SEC, 704 cow_time_ns % NSEC_PER_SEC); 705 ksft_print_msg("Average speed: %.3f MiB/s\n\n", ((page_size * (page_count / 2)) / MB) / 706 ((double)cow_time_ns / NSEC_PER_SEC)); 707 708 /* Create 2000 pairs of duplicate pages */ 709 for (size_t i = 0; i < page_count - 1; i = i + 2) { 710 memset(map_ptr + page_size * i, '+', i / 2 + 1); 711 memset(map_ptr + page_size * (i + 1), '+', i / 2 + 1); 712 } 713 if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout)) 714 goto err_out; 715 716 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 717 ksft_perror("clock_gettime"); 718 goto err_out; 719 } 720 for (size_t i = 0; i < page_count - 1; i = i + 2) 721 memset(map_ptr + page_size * i, '-', 1); 722 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { 723 ksft_perror("clock_gettime"); 724 goto err_out; 725 } 726 727 cow_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + 728 (end_time.tv_nsec - start_time.tv_nsec); 729 730 ksft_print_msg("Merged pages:\n"); 731 ksft_print_msg("Total time: %ld.%09ld s\n", cow_time_ns / NSEC_PER_SEC, 732 cow_time_ns % NSEC_PER_SEC); 733 ksft_print_msg("Average speed: %.3f MiB/s\n", ((page_size * (page_count / 2)) / MB) / 734 ((double)cow_time_ns / NSEC_PER_SEC)); 735 736 munmap(map_ptr, page_size * page_count); 737 return KSFT_PASS; 738 739 err_out: 740 munmap(map_ptr, page_size * page_count); 741 return KSFT_FAIL; 742 } 743 744 int main(int argc, char *argv[]) 745 { 746 int ret = 0, opt; 747 int prot = 0; 748 int ksm_scan_limit_sec = KSM_SCAN_LIMIT_SEC_DEFAULT; 749 int merge_type = KSM_MERGE_TYPE_DEFAULT; 750 long page_count = KSM_PAGE_COUNT_DEFAULT; 751 size_t page_size = sysconf(_SC_PAGESIZE); 752 struct ksm_sysfs ksm_sysfs_old; 753 int test_name = CHECK_KSM_MERGE; 754 bool use_zero_pages = KSM_USE_ZERO_PAGES_DEFAULT; 755 bool merge_across_nodes = KSM_MERGE_ACROSS_NODES_DEFAULT; 756 long size_MB = 0; 757 const char *test_descr = "KSM merging"; 758 759 ksft_print_header(); 760 ksft_set_plan(1); 761 762 while ((opt = getopt(argc, argv, "dha:p:l:z:m:s:t:MUZNPCHD")) != -1) { 763 switch (opt) { 764 case 'a': 765 prot = str_to_prot(optarg); 766 break; 767 case 'p': 768 page_count = atol(optarg); 769 if (page_count <= 0) 770 ksft_exit_fail_msg("The number of pages must be greater than 0\n"); 771 break; 772 case 'l': 773 ksm_scan_limit_sec = atoi(optarg); 774 if (ksm_scan_limit_sec <= 0) 775 ksft_exit_fail_msg("Timeout value must be greater than 0\n"); 776 break; 777 case 'h': 778 print_help(); 779 break; 780 case 'z': 781 if (strcmp(optarg, "0") == 0) 782 use_zero_pages = 0; 783 else 784 use_zero_pages = 1; 785 break; 786 case 'm': 787 if (strcmp(optarg, "0") == 0) 788 merge_across_nodes = 0; 789 else 790 merge_across_nodes = 1; 791 break; 792 case 'd': 793 debug = 1; 794 break; 795 case 's': 796 size_MB = atoi(optarg); 797 if (size_MB <= 0) 798 ksft_exit_fail_msg("Size must be greater than 0\n"); 799 break; 800 case 't': 801 { 802 int tmp = atoi(optarg); 803 804 if (tmp < 0 || tmp > KSM_MERGE_LAST) 805 ksft_exit_fail_msg("Invalid merge type\n"); 806 merge_type = tmp; 807 } 808 break; 809 case 'M': 810 break; 811 case 'U': 812 test_name = CHECK_KSM_UNMERGE; 813 break; 814 case 'Z': 815 test_name = CHECK_KSM_ZERO_PAGE_MERGE; 816 break; 817 case 'N': 818 test_name = CHECK_KSM_NUMA_MERGE; 819 break; 820 case 'P': 821 test_name = KSM_MERGE_TIME; 822 break; 823 case 'H': 824 test_name = KSM_MERGE_TIME_HUGE_PAGES; 825 break; 826 case 'D': 827 test_name = KSM_UNMERGE_TIME; 828 break; 829 case 'C': 830 test_name = KSM_COW_TIME; 831 break; 832 default: 833 ksft_exit_fail_msg("Unknown option\n"); 834 } 835 } 836 837 if (prot == 0) 838 prot = str_to_prot(KSM_PROT_STR_DEFAULT); 839 840 if (access(KSM_SYSFS_PATH, F_OK)) 841 ksft_exit_skip("Config KSM not enabled\n"); 842 843 if (ksm_save_def(&ksm_sysfs_old)) 844 ksft_exit_fail_msg("Cannot save default tunables\n"); 845 846 if (ksm_write_sysfs(KSM_FP("run"), 2) || 847 ksm_write_sysfs(KSM_FP("sleep_millisecs"), 0) || 848 (numa_available() ? 0 : 849 ksm_write_sysfs(KSM_FP("merge_across_nodes"), 1)) || 850 ksm_write_sysfs(KSM_FP("pages_to_scan"), page_count)) 851 ksft_exit_fail_msg("Cannot set up KSM tunables\n"); 852 853 switch (test_name) { 854 case CHECK_KSM_MERGE: 855 test_descr = "KSM merging"; 856 ret = check_ksm_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, page_count, 857 ksm_scan_limit_sec, page_size); 858 break; 859 case CHECK_KSM_UNMERGE: 860 test_descr = "KSM unmerging"; 861 ret = check_ksm_unmerge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, 862 ksm_scan_limit_sec, page_size); 863 break; 864 case CHECK_KSM_ZERO_PAGE_MERGE: 865 test_descr = "KSM zero page merging"; 866 ret = check_ksm_zero_page_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, 867 page_count, ksm_scan_limit_sec, use_zero_pages, 868 page_size); 869 break; 870 case CHECK_KSM_NUMA_MERGE: 871 test_descr = "KSM NUMA merging"; 872 ret = check_ksm_numa_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, 873 ksm_scan_limit_sec, merge_across_nodes, page_size); 874 break; 875 case KSM_MERGE_TIME: 876 if (size_MB == 0) 877 ksft_exit_fail_msg("Option '-s' is required\n"); 878 test_descr = "KSM merge time"; 879 ret = ksm_merge_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, 880 ksm_scan_limit_sec, size_MB); 881 break; 882 case KSM_MERGE_TIME_HUGE_PAGES: 883 if (size_MB == 0) 884 ksft_exit_fail_msg("Option '-s' is required\n"); 885 test_descr = "KSM merge time with huge pages"; 886 ret = ksm_merge_hugepages_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, 887 ksm_scan_limit_sec, size_MB); 888 break; 889 case KSM_UNMERGE_TIME: 890 if (size_MB == 0) 891 ksft_exit_fail_msg("Option '-s' is required\n"); 892 test_descr = "KSM unmerge time"; 893 ret = ksm_unmerge_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, 894 ksm_scan_limit_sec, size_MB); 895 break; 896 case KSM_COW_TIME: 897 test_descr = "KSM COW time"; 898 ret = ksm_cow_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, 899 ksm_scan_limit_sec, page_size); 900 break; 901 } 902 903 if (ksm_restore(&ksm_sysfs_old)) 904 ksft_print_msg("Cannot restore default tunables\n"); 905 906 ksft_test_result_report(ret, "%s\n", test_descr); 907 908 ksft_finished(); 909 } 910