1 // SPDX-License-Identifier: GPL-2.0
2
3 #include <sys/mman.h>
4 #include <sys/prctl.h>
5 #include <sys/wait.h>
6 #include <stdbool.h>
7 #include <time.h>
8 #include <string.h>
9 #include <numa.h>
10 #include <unistd.h>
11 #include <fcntl.h>
12 #include <stdint.h>
13 #include <err.h>
14
15 #include "../kselftest.h"
16 #include <include/vdso/time64.h>
17 #include "vm_util.h"
18 #include "thp_settings.h"
19
20 #define KSM_SYSFS_PATH "/sys/kernel/mm/ksm/"
21 #define KSM_FP(s) (KSM_SYSFS_PATH s)
22 #define KSM_SCAN_LIMIT_SEC_DEFAULT 120
23 #define KSM_PAGE_COUNT_DEFAULT 10l
24 #define KSM_PROT_STR_DEFAULT "rw"
25 #define KSM_USE_ZERO_PAGES_DEFAULT false
26 #define KSM_MERGE_ACROSS_NODES_DEFAULT true
27 #define KSM_MERGE_TYPE_DEFAULT 0
28 #define MB (1ul << 20)
29
30 struct ksm_sysfs {
31 unsigned long max_page_sharing;
32 unsigned long merge_across_nodes;
33 unsigned long pages_to_scan;
34 unsigned long run;
35 unsigned long sleep_millisecs;
36 unsigned long stable_node_chains_prune_millisecs;
37 unsigned long use_zero_pages;
38 };
39
40 enum ksm_merge_type {
41 KSM_MERGE_MADVISE,
42 KSM_MERGE_PRCTL,
43 KSM_MERGE_LAST = KSM_MERGE_PRCTL
44 };
45
46 enum ksm_test_name {
47 CHECK_KSM_MERGE,
48 CHECK_KSM_UNMERGE,
49 CHECK_KSM_GET_MERGE_TYPE,
50 CHECK_KSM_ZERO_PAGE_MERGE,
51 CHECK_KSM_NUMA_MERGE,
52 KSM_MERGE_TIME,
53 KSM_MERGE_TIME_HUGE_PAGES,
54 KSM_UNMERGE_TIME,
55 KSM_COW_TIME
56 };
57
58 int debug;
59
ksm_write_sysfs(const char * file_path,unsigned long val)60 static int ksm_write_sysfs(const char *file_path, unsigned long val)
61 {
62 return write_sysfs(file_path, val);
63 }
64
ksm_read_sysfs(const char * file_path,unsigned long * val)65 static int ksm_read_sysfs(const char *file_path, unsigned long *val)
66 {
67 return read_sysfs(file_path, val);
68 }
69
ksm_print_sysfs(void)70 static void ksm_print_sysfs(void)
71 {
72 unsigned long max_page_sharing, pages_sharing, pages_shared;
73 unsigned long full_scans, pages_unshared, pages_volatile;
74 unsigned long stable_node_chains, stable_node_dups;
75 long general_profit;
76
77 if (ksm_read_sysfs(KSM_FP("pages_shared"), &pages_shared) ||
78 ksm_read_sysfs(KSM_FP("pages_sharing"), &pages_sharing) ||
79 ksm_read_sysfs(KSM_FP("max_page_sharing"), &max_page_sharing) ||
80 ksm_read_sysfs(KSM_FP("full_scans"), &full_scans) ||
81 ksm_read_sysfs(KSM_FP("pages_unshared"), &pages_unshared) ||
82 ksm_read_sysfs(KSM_FP("pages_volatile"), &pages_volatile) ||
83 ksm_read_sysfs(KSM_FP("stable_node_chains"), &stable_node_chains) ||
84 ksm_read_sysfs(KSM_FP("stable_node_dups"), &stable_node_dups) ||
85 ksm_read_sysfs(KSM_FP("general_profit"), (unsigned long *)&general_profit))
86 return;
87
88 printf("pages_shared : %lu\n", pages_shared);
89 printf("pages_sharing : %lu\n", pages_sharing);
90 printf("max_page_sharing : %lu\n", max_page_sharing);
91 printf("full_scans : %lu\n", full_scans);
92 printf("pages_unshared : %lu\n", pages_unshared);
93 printf("pages_volatile : %lu\n", pages_volatile);
94 printf("stable_node_chains: %lu\n", stable_node_chains);
95 printf("stable_node_dups : %lu\n", stable_node_dups);
96 printf("general_profit : %ld\n", general_profit);
97 }
98
ksm_print_procfs(void)99 static void ksm_print_procfs(void)
100 {
101 const char *file_name = "/proc/self/ksm_stat";
102 char buffer[512];
103 FILE *f = fopen(file_name, "r");
104
105 if (!f) {
106 fprintf(stderr, "f %s\n", file_name);
107 perror("fopen");
108 return;
109 }
110
111 while (fgets(buffer, sizeof(buffer), f))
112 printf("%s", buffer);
113
114 fclose(f);
115 }
116
str_to_prot(char * prot_str)117 static int str_to_prot(char *prot_str)
118 {
119 int prot = 0;
120
121 if ((strchr(prot_str, 'r')) != NULL)
122 prot |= PROT_READ;
123 if ((strchr(prot_str, 'w')) != NULL)
124 prot |= PROT_WRITE;
125 if ((strchr(prot_str, 'x')) != NULL)
126 prot |= PROT_EXEC;
127
128 return prot;
129 }
130
print_help(void)131 static void print_help(void)
132 {
133 printf("usage: ksm_tests [-h] <test type> [-a prot] [-p page_count] [-l timeout]\n"
134 "[-z use_zero_pages] [-m merge_across_nodes] [-s size]\n");
135
136 printf("Supported <test type>:\n"
137 " -M (page merging)\n"
138 " -Z (zero pages merging)\n"
139 " -N (merging of pages in different NUMA nodes)\n"
140 " -U (page unmerging)\n"
141 " -P evaluate merging time and speed.\n"
142 " For this test, the size of duplicated memory area (in MiB)\n"
143 " must be provided using -s option\n"
144 " -H evaluate merging time and speed of area allocated mostly with huge pages\n"
145 " For this test, the size of duplicated memory area (in MiB)\n"
146 " must be provided using -s option\n"
147 " -D evaluate unmerging time and speed when disabling KSM.\n"
148 " For this test, the size of duplicated memory area (in MiB)\n"
149 " must be provided using -s option\n"
150 " -C evaluate the time required to break COW of merged pages.\n\n");
151
152 printf(" -a: specify the access protections of pages.\n"
153 " <prot> must be of the form [rwx].\n"
154 " Default: %s\n", KSM_PROT_STR_DEFAULT);
155 printf(" -p: specify the number of pages to test.\n"
156 " Default: %ld\n", KSM_PAGE_COUNT_DEFAULT);
157 printf(" -l: limit the maximum running time (in seconds) for a test.\n"
158 " Default: %d seconds\n", KSM_SCAN_LIMIT_SEC_DEFAULT);
159 printf(" -z: change use_zero_pages tunable\n"
160 " Default: %d\n", KSM_USE_ZERO_PAGES_DEFAULT);
161 printf(" -m: change merge_across_nodes tunable\n"
162 " Default: %d\n", KSM_MERGE_ACROSS_NODES_DEFAULT);
163 printf(" -d: turn debugging output on\n");
164 printf(" -s: the size of duplicated memory area (in MiB)\n");
165 printf(" -t: KSM merge type\n"
166 " Default: 0\n"
167 " 0: madvise merging\n"
168 " 1: prctl merging\n");
169
170 exit(0);
171 }
172
allocate_memory(void * ptr,int prot,int mapping,char data,size_t map_size)173 static void *allocate_memory(void *ptr, int prot, int mapping, char data, size_t map_size)
174 {
175 void *map_ptr = mmap(ptr, map_size, PROT_WRITE, mapping, -1, 0);
176
177 if (!map_ptr) {
178 perror("mmap");
179 return NULL;
180 }
181 memset(map_ptr, data, map_size);
182 if (mprotect(map_ptr, map_size, prot)) {
183 perror("mprotect");
184 munmap(map_ptr, map_size);
185 return NULL;
186 }
187
188 return map_ptr;
189 }
190
ksm_do_scan(int scan_count,struct timespec start_time,int timeout)191 static int ksm_do_scan(int scan_count, struct timespec start_time, int timeout)
192 {
193 struct timespec cur_time;
194 unsigned long cur_scan, init_scan;
195
196 if (ksm_read_sysfs(KSM_FP("full_scans"), &init_scan))
197 return 1;
198 cur_scan = init_scan;
199
200 while (cur_scan < init_scan + scan_count) {
201 if (ksm_read_sysfs(KSM_FP("full_scans"), &cur_scan))
202 return 1;
203 if (clock_gettime(CLOCK_MONOTONIC_RAW, &cur_time)) {
204 perror("clock_gettime");
205 return 1;
206 }
207 if ((cur_time.tv_sec - start_time.tv_sec) > timeout) {
208 printf("Scan time limit exceeded\n");
209 return 1;
210 }
211 }
212
213 return 0;
214 }
215
ksm_merge_pages(int merge_type,void * addr,size_t size,struct timespec start_time,int timeout)216 static int ksm_merge_pages(int merge_type, void *addr, size_t size,
217 struct timespec start_time, int timeout)
218 {
219 if (merge_type == KSM_MERGE_MADVISE) {
220 if (madvise(addr, size, MADV_MERGEABLE)) {
221 perror("madvise");
222 return 1;
223 }
224 } else if (merge_type == KSM_MERGE_PRCTL) {
225 if (prctl(PR_SET_MEMORY_MERGE, 1, 0, 0, 0)) {
226 perror("prctl");
227 return 1;
228 }
229 }
230
231 if (ksm_write_sysfs(KSM_FP("run"), 1))
232 return 1;
233
234 /* Since merging occurs only after 2 scans, make sure to get at least 2 full scans */
235 if (ksm_do_scan(2, start_time, timeout))
236 return 1;
237
238 return 0;
239 }
240
ksm_unmerge_pages(void * addr,size_t size,struct timespec start_time,int timeout)241 static int ksm_unmerge_pages(void *addr, size_t size,
242 struct timespec start_time, int timeout)
243 {
244 if (madvise(addr, size, MADV_UNMERGEABLE)) {
245 perror("madvise");
246 return 1;
247 }
248 return 0;
249 }
250
assert_ksm_pages_count(long dupl_page_count)251 static bool assert_ksm_pages_count(long dupl_page_count)
252 {
253 unsigned long max_page_sharing, pages_sharing, pages_shared;
254
255 if (ksm_read_sysfs(KSM_FP("pages_shared"), &pages_shared) ||
256 ksm_read_sysfs(KSM_FP("pages_sharing"), &pages_sharing) ||
257 ksm_read_sysfs(KSM_FP("max_page_sharing"), &max_page_sharing))
258 return false;
259
260 if (debug) {
261 ksm_print_sysfs();
262 ksm_print_procfs();
263 }
264
265 /*
266 * Since there must be at least 2 pages for merging and 1 page can be
267 * shared with the limited number of pages (max_page_sharing), sometimes
268 * there are 'leftover' pages that cannot be merged. For example, if there
269 * are 11 pages and max_page_sharing = 10, then only 10 pages will be
270 * merged and the 11th page won't be affected. As a result, when the number
271 * of duplicate pages is divided by max_page_sharing and the remainder is 1,
272 * pages_shared and pages_sharing values will be equal between dupl_page_count
273 * and dupl_page_count - 1.
274 */
275 if (dupl_page_count % max_page_sharing == 1 || dupl_page_count % max_page_sharing == 0) {
276 if (pages_shared == dupl_page_count / max_page_sharing &&
277 pages_sharing == pages_shared * (max_page_sharing - 1))
278 return true;
279 } else {
280 if (pages_shared == (dupl_page_count / max_page_sharing + 1) &&
281 pages_sharing == dupl_page_count - pages_shared)
282 return true;
283 }
284
285 return false;
286 }
287
ksm_save_def(struct ksm_sysfs * ksm_sysfs)288 static int ksm_save_def(struct ksm_sysfs *ksm_sysfs)
289 {
290 if (ksm_read_sysfs(KSM_FP("max_page_sharing"), &ksm_sysfs->max_page_sharing) ||
291 numa_available() ? 0 :
292 ksm_read_sysfs(KSM_FP("merge_across_nodes"), &ksm_sysfs->merge_across_nodes) ||
293 ksm_read_sysfs(KSM_FP("sleep_millisecs"), &ksm_sysfs->sleep_millisecs) ||
294 ksm_read_sysfs(KSM_FP("pages_to_scan"), &ksm_sysfs->pages_to_scan) ||
295 ksm_read_sysfs(KSM_FP("run"), &ksm_sysfs->run) ||
296 ksm_read_sysfs(KSM_FP("stable_node_chains_prune_millisecs"),
297 &ksm_sysfs->stable_node_chains_prune_millisecs) ||
298 ksm_read_sysfs(KSM_FP("use_zero_pages"), &ksm_sysfs->use_zero_pages))
299 return 1;
300
301 return 0;
302 }
303
ksm_restore(struct ksm_sysfs * ksm_sysfs)304 static int ksm_restore(struct ksm_sysfs *ksm_sysfs)
305 {
306 if (ksm_write_sysfs(KSM_FP("max_page_sharing"), ksm_sysfs->max_page_sharing) ||
307 numa_available() ? 0 :
308 ksm_write_sysfs(KSM_FP("merge_across_nodes"), ksm_sysfs->merge_across_nodes) ||
309 ksm_write_sysfs(KSM_FP("pages_to_scan"), ksm_sysfs->pages_to_scan) ||
310 ksm_write_sysfs(KSM_FP("run"), ksm_sysfs->run) ||
311 ksm_write_sysfs(KSM_FP("sleep_millisecs"), ksm_sysfs->sleep_millisecs) ||
312 ksm_write_sysfs(KSM_FP("stable_node_chains_prune_millisecs"),
313 ksm_sysfs->stable_node_chains_prune_millisecs) ||
314 ksm_write_sysfs(KSM_FP("use_zero_pages"), ksm_sysfs->use_zero_pages))
315 return 1;
316
317 return 0;
318 }
319
check_ksm_merge(int merge_type,int mapping,int prot,long page_count,int timeout,size_t page_size)320 static int check_ksm_merge(int merge_type, int mapping, int prot,
321 long page_count, int timeout, size_t page_size)
322 {
323 void *map_ptr;
324 struct timespec start_time;
325
326 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
327 perror("clock_gettime");
328 return KSFT_FAIL;
329 }
330
331 /* fill pages with the same data and merge them */
332 map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count);
333 if (!map_ptr)
334 return KSFT_FAIL;
335
336 if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout))
337 goto err_out;
338
339 /* verify that the right number of pages are merged */
340 if (assert_ksm_pages_count(page_count)) {
341 printf("OK\n");
342 munmap(map_ptr, page_size * page_count);
343 if (merge_type == KSM_MERGE_PRCTL)
344 prctl(PR_SET_MEMORY_MERGE, 0, 0, 0, 0);
345 return KSFT_PASS;
346 }
347
348 err_out:
349 printf("Not OK\n");
350 munmap(map_ptr, page_size * page_count);
351 return KSFT_FAIL;
352 }
353
check_ksm_unmerge(int merge_type,int mapping,int prot,int timeout,size_t page_size)354 static int check_ksm_unmerge(int merge_type, int mapping, int prot, int timeout, size_t page_size)
355 {
356 void *map_ptr;
357 struct timespec start_time;
358 int page_count = 2;
359
360 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
361 perror("clock_gettime");
362 return KSFT_FAIL;
363 }
364
365 /* fill pages with the same data and merge them */
366 map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count);
367 if (!map_ptr)
368 return KSFT_FAIL;
369
370 if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout))
371 goto err_out;
372
373 /* change 1 byte in each of the 2 pages -- KSM must automatically unmerge them */
374 memset(map_ptr, '-', 1);
375 memset(map_ptr + page_size, '+', 1);
376
377 /* get at least 1 scan, so KSM can detect that the pages were modified */
378 if (ksm_do_scan(1, start_time, timeout))
379 goto err_out;
380
381 /* check that unmerging was successful and 0 pages are currently merged */
382 if (assert_ksm_pages_count(0)) {
383 printf("OK\n");
384 munmap(map_ptr, page_size * page_count);
385 return KSFT_PASS;
386 }
387
388 err_out:
389 printf("Not OK\n");
390 munmap(map_ptr, page_size * page_count);
391 return KSFT_FAIL;
392 }
393
check_ksm_zero_page_merge(int merge_type,int mapping,int prot,long page_count,int timeout,bool use_zero_pages,size_t page_size)394 static int check_ksm_zero_page_merge(int merge_type, int mapping, int prot, long page_count,
395 int timeout, bool use_zero_pages, size_t page_size)
396 {
397 void *map_ptr;
398 struct timespec start_time;
399
400 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
401 perror("clock_gettime");
402 return KSFT_FAIL;
403 }
404
405 if (ksm_write_sysfs(KSM_FP("use_zero_pages"), use_zero_pages))
406 return KSFT_FAIL;
407
408 /* fill pages with zero and try to merge them */
409 map_ptr = allocate_memory(NULL, prot, mapping, 0, page_size * page_count);
410 if (!map_ptr)
411 return KSFT_FAIL;
412
413 if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout))
414 goto err_out;
415
416 /*
417 * verify that the right number of pages are merged:
418 * 1) if use_zero_pages is set to 1, empty pages are merged
419 * with the kernel zero page instead of with each other;
420 * 2) if use_zero_pages is set to 0, empty pages are not treated specially
421 * and merged as usual.
422 */
423 if (use_zero_pages && !assert_ksm_pages_count(0))
424 goto err_out;
425 else if (!use_zero_pages && !assert_ksm_pages_count(page_count))
426 goto err_out;
427
428 printf("OK\n");
429 munmap(map_ptr, page_size * page_count);
430 return KSFT_PASS;
431
432 err_out:
433 printf("Not OK\n");
434 munmap(map_ptr, page_size * page_count);
435 return KSFT_FAIL;
436 }
437
get_next_mem_node(int node)438 static int get_next_mem_node(int node)
439 {
440
441 long node_size;
442 int mem_node = 0;
443 int i, max_node = numa_max_node();
444
445 for (i = node + 1; i <= max_node + node; i++) {
446 mem_node = i % (max_node + 1);
447 node_size = numa_node_size(mem_node, NULL);
448 if (node_size > 0)
449 break;
450 }
451 return mem_node;
452 }
453
get_first_mem_node(void)454 static int get_first_mem_node(void)
455 {
456 return get_next_mem_node(numa_max_node());
457 }
458
check_ksm_numa_merge(int merge_type,int mapping,int prot,int timeout,bool merge_across_nodes,size_t page_size)459 static int check_ksm_numa_merge(int merge_type, int mapping, int prot, int timeout,
460 bool merge_across_nodes, size_t page_size)
461 {
462 void *numa1_map_ptr, *numa2_map_ptr;
463 struct timespec start_time;
464 int page_count = 2;
465 int first_node;
466
467 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
468 perror("clock_gettime");
469 return KSFT_FAIL;
470 }
471
472 if (numa_available() < 0) {
473 perror("NUMA support not enabled");
474 return KSFT_SKIP;
475 }
476 if (numa_num_configured_nodes() <= 1) {
477 printf("At least 2 NUMA nodes must be available\n");
478 return KSFT_SKIP;
479 }
480 if (ksm_write_sysfs(KSM_FP("merge_across_nodes"), merge_across_nodes))
481 return KSFT_FAIL;
482
483 /* allocate 2 pages in 2 different NUMA nodes and fill them with the same data */
484 first_node = get_first_mem_node();
485 numa1_map_ptr = numa_alloc_onnode(page_size, first_node);
486 numa2_map_ptr = numa_alloc_onnode(page_size, get_next_mem_node(first_node));
487 if (!numa1_map_ptr || !numa2_map_ptr) {
488 perror("numa_alloc_onnode");
489 return KSFT_FAIL;
490 }
491
492 memset(numa1_map_ptr, '*', page_size);
493 memset(numa2_map_ptr, '*', page_size);
494
495 /* try to merge the pages */
496 if (ksm_merge_pages(merge_type, numa1_map_ptr, page_size, start_time, timeout) ||
497 ksm_merge_pages(merge_type, numa2_map_ptr, page_size, start_time, timeout))
498 goto err_out;
499
500 /*
501 * verify that the right number of pages are merged:
502 * 1) if merge_across_nodes was enabled, 2 duplicate pages will be merged;
503 * 2) if merge_across_nodes = 0, there must be 0 merged pages, since there is
504 * only 1 unique page in each node and they can't be shared.
505 */
506 if (merge_across_nodes && !assert_ksm_pages_count(page_count))
507 goto err_out;
508 else if (!merge_across_nodes && !assert_ksm_pages_count(0))
509 goto err_out;
510
511 numa_free(numa1_map_ptr, page_size);
512 numa_free(numa2_map_ptr, page_size);
513 printf("OK\n");
514 return KSFT_PASS;
515
516 err_out:
517 numa_free(numa1_map_ptr, page_size);
518 numa_free(numa2_map_ptr, page_size);
519 printf("Not OK\n");
520 return KSFT_FAIL;
521 }
522
ksm_merge_hugepages_time(int merge_type,int mapping,int prot,int timeout,size_t map_size)523 static int ksm_merge_hugepages_time(int merge_type, int mapping, int prot,
524 int timeout, size_t map_size)
525 {
526 void *map_ptr, *map_ptr_orig;
527 struct timespec start_time, end_time;
528 unsigned long scan_time_ns;
529 int pagemap_fd, n_normal_pages, n_huge_pages;
530
531 if (!thp_is_enabled()) {
532 printf("Transparent Hugepages not available\n");
533 return KSFT_SKIP;
534 }
535
536 map_size *= MB;
537 size_t len = map_size;
538
539 len -= len % HPAGE_SIZE;
540 map_ptr_orig = mmap(NULL, len + HPAGE_SIZE, PROT_READ | PROT_WRITE,
541 MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE, -1, 0);
542 map_ptr = map_ptr_orig + HPAGE_SIZE - (uintptr_t)map_ptr_orig % HPAGE_SIZE;
543
544 if (map_ptr_orig == MAP_FAILED)
545 err(2, "initial mmap");
546
547 if (madvise(map_ptr, len, MADV_HUGEPAGE))
548 err(2, "MADV_HUGEPAGE");
549
550 pagemap_fd = open("/proc/self/pagemap", O_RDONLY);
551 if (pagemap_fd < 0)
552 err(2, "open pagemap");
553
554 n_normal_pages = 0;
555 n_huge_pages = 0;
556 for (void *p = map_ptr; p < map_ptr + len; p += HPAGE_SIZE) {
557 if (allocate_transhuge(p, pagemap_fd) < 0)
558 n_normal_pages++;
559 else
560 n_huge_pages++;
561 }
562 printf("Number of normal pages: %d\n", n_normal_pages);
563 printf("Number of huge pages: %d\n", n_huge_pages);
564
565 memset(map_ptr, '*', len);
566
567 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
568 perror("clock_gettime");
569 goto err_out;
570 }
571 if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout))
572 goto err_out;
573 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) {
574 perror("clock_gettime");
575 goto err_out;
576 }
577
578 scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC +
579 (end_time.tv_nsec - start_time.tv_nsec);
580
581 printf("Total size: %lu MiB\n", map_size / MB);
582 printf("Total time: %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC,
583 scan_time_ns % NSEC_PER_SEC);
584 printf("Average speed: %.3f MiB/s\n", (map_size / MB) /
585 ((double)scan_time_ns / NSEC_PER_SEC));
586
587 munmap(map_ptr_orig, len + HPAGE_SIZE);
588 return KSFT_PASS;
589
590 err_out:
591 printf("Not OK\n");
592 munmap(map_ptr_orig, len + HPAGE_SIZE);
593 return KSFT_FAIL;
594 }
595
ksm_merge_time(int merge_type,int mapping,int prot,int timeout,size_t map_size)596 static int ksm_merge_time(int merge_type, int mapping, int prot, int timeout, size_t map_size)
597 {
598 void *map_ptr;
599 struct timespec start_time, end_time;
600 unsigned long scan_time_ns;
601
602 map_size *= MB;
603
604 map_ptr = allocate_memory(NULL, prot, mapping, '*', map_size);
605 if (!map_ptr)
606 return KSFT_FAIL;
607
608 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
609 perror("clock_gettime");
610 goto err_out;
611 }
612 if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout))
613 goto err_out;
614 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) {
615 perror("clock_gettime");
616 goto err_out;
617 }
618
619 scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC +
620 (end_time.tv_nsec - start_time.tv_nsec);
621
622 printf("Total size: %lu MiB\n", map_size / MB);
623 printf("Total time: %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC,
624 scan_time_ns % NSEC_PER_SEC);
625 printf("Average speed: %.3f MiB/s\n", (map_size / MB) /
626 ((double)scan_time_ns / NSEC_PER_SEC));
627
628 munmap(map_ptr, map_size);
629 return KSFT_PASS;
630
631 err_out:
632 printf("Not OK\n");
633 munmap(map_ptr, map_size);
634 return KSFT_FAIL;
635 }
636
ksm_unmerge_time(int merge_type,int mapping,int prot,int timeout,size_t map_size)637 static int ksm_unmerge_time(int merge_type, int mapping, int prot, int timeout, size_t map_size)
638 {
639 void *map_ptr;
640 struct timespec start_time, end_time;
641 unsigned long scan_time_ns;
642
643 map_size *= MB;
644
645 map_ptr = allocate_memory(NULL, prot, mapping, '*', map_size);
646 if (!map_ptr)
647 return KSFT_FAIL;
648 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
649 perror("clock_gettime");
650 goto err_out;
651 }
652 if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout))
653 goto err_out;
654
655 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
656 perror("clock_gettime");
657 goto err_out;
658 }
659 if (ksm_unmerge_pages(map_ptr, map_size, start_time, timeout))
660 goto err_out;
661 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) {
662 perror("clock_gettime");
663 goto err_out;
664 }
665
666 scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC +
667 (end_time.tv_nsec - start_time.tv_nsec);
668
669 printf("Total size: %lu MiB\n", map_size / MB);
670 printf("Total time: %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC,
671 scan_time_ns % NSEC_PER_SEC);
672 printf("Average speed: %.3f MiB/s\n", (map_size / MB) /
673 ((double)scan_time_ns / NSEC_PER_SEC));
674
675 munmap(map_ptr, map_size);
676 return KSFT_PASS;
677
678 err_out:
679 printf("Not OK\n");
680 munmap(map_ptr, map_size);
681 return KSFT_FAIL;
682 }
683
ksm_cow_time(int merge_type,int mapping,int prot,int timeout,size_t page_size)684 static int ksm_cow_time(int merge_type, int mapping, int prot, int timeout, size_t page_size)
685 {
686 void *map_ptr;
687 struct timespec start_time, end_time;
688 unsigned long cow_time_ns;
689
690 /* page_count must be less than 2*page_size */
691 size_t page_count = 4000;
692
693 map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count);
694 if (!map_ptr)
695 return KSFT_FAIL;
696
697 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
698 perror("clock_gettime");
699 return KSFT_FAIL;
700 }
701 for (size_t i = 0; i < page_count - 1; i = i + 2)
702 memset(map_ptr + page_size * i, '-', 1);
703 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) {
704 perror("clock_gettime");
705 return KSFT_FAIL;
706 }
707
708 cow_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC +
709 (end_time.tv_nsec - start_time.tv_nsec);
710
711 printf("Total size: %lu MiB\n\n", (page_size * page_count) / MB);
712 printf("Not merged pages:\n");
713 printf("Total time: %ld.%09ld s\n", cow_time_ns / NSEC_PER_SEC,
714 cow_time_ns % NSEC_PER_SEC);
715 printf("Average speed: %.3f MiB/s\n\n", ((page_size * (page_count / 2)) / MB) /
716 ((double)cow_time_ns / NSEC_PER_SEC));
717
718 /* Create 2000 pairs of duplicate pages */
719 for (size_t i = 0; i < page_count - 1; i = i + 2) {
720 memset(map_ptr + page_size * i, '+', i / 2 + 1);
721 memset(map_ptr + page_size * (i + 1), '+', i / 2 + 1);
722 }
723 if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout))
724 goto err_out;
725
726 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
727 perror("clock_gettime");
728 goto err_out;
729 }
730 for (size_t i = 0; i < page_count - 1; i = i + 2)
731 memset(map_ptr + page_size * i, '-', 1);
732 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) {
733 perror("clock_gettime");
734 goto err_out;
735 }
736
737 cow_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC +
738 (end_time.tv_nsec - start_time.tv_nsec);
739
740 printf("Merged pages:\n");
741 printf("Total time: %ld.%09ld s\n", cow_time_ns / NSEC_PER_SEC,
742 cow_time_ns % NSEC_PER_SEC);
743 printf("Average speed: %.3f MiB/s\n", ((page_size * (page_count / 2)) / MB) /
744 ((double)cow_time_ns / NSEC_PER_SEC));
745
746 munmap(map_ptr, page_size * page_count);
747 return KSFT_PASS;
748
749 err_out:
750 printf("Not OK\n");
751 munmap(map_ptr, page_size * page_count);
752 return KSFT_FAIL;
753 }
754
main(int argc,char * argv[])755 int main(int argc, char *argv[])
756 {
757 int ret = 0, opt;
758 int prot = 0;
759 int ksm_scan_limit_sec = KSM_SCAN_LIMIT_SEC_DEFAULT;
760 int merge_type = KSM_MERGE_TYPE_DEFAULT;
761 long page_count = KSM_PAGE_COUNT_DEFAULT;
762 size_t page_size = sysconf(_SC_PAGESIZE);
763 struct ksm_sysfs ksm_sysfs_old;
764 int test_name = CHECK_KSM_MERGE;
765 bool use_zero_pages = KSM_USE_ZERO_PAGES_DEFAULT;
766 bool merge_across_nodes = KSM_MERGE_ACROSS_NODES_DEFAULT;
767 long size_MB = 0;
768
769 while ((opt = getopt(argc, argv, "dha:p:l:z:m:s:t:MUZNPCHD")) != -1) {
770 switch (opt) {
771 case 'a':
772 prot = str_to_prot(optarg);
773 break;
774 case 'p':
775 page_count = atol(optarg);
776 if (page_count <= 0) {
777 printf("The number of pages must be greater than 0\n");
778 return KSFT_FAIL;
779 }
780 break;
781 case 'l':
782 ksm_scan_limit_sec = atoi(optarg);
783 if (ksm_scan_limit_sec <= 0) {
784 printf("Timeout value must be greater than 0\n");
785 return KSFT_FAIL;
786 }
787 break;
788 case 'h':
789 print_help();
790 break;
791 case 'z':
792 if (strcmp(optarg, "0") == 0)
793 use_zero_pages = 0;
794 else
795 use_zero_pages = 1;
796 break;
797 case 'm':
798 if (strcmp(optarg, "0") == 0)
799 merge_across_nodes = 0;
800 else
801 merge_across_nodes = 1;
802 break;
803 case 'd':
804 debug = 1;
805 break;
806 case 's':
807 size_MB = atoi(optarg);
808 if (size_MB <= 0) {
809 printf("Size must be greater than 0\n");
810 return KSFT_FAIL;
811 }
812 break;
813 case 't':
814 {
815 int tmp = atoi(optarg);
816
817 if (tmp < 0 || tmp > KSM_MERGE_LAST) {
818 printf("Invalid merge type\n");
819 return KSFT_FAIL;
820 }
821 merge_type = tmp;
822 }
823 break;
824 case 'M':
825 break;
826 case 'U':
827 test_name = CHECK_KSM_UNMERGE;
828 break;
829 case 'Z':
830 test_name = CHECK_KSM_ZERO_PAGE_MERGE;
831 break;
832 case 'N':
833 test_name = CHECK_KSM_NUMA_MERGE;
834 break;
835 case 'P':
836 test_name = KSM_MERGE_TIME;
837 break;
838 case 'H':
839 test_name = KSM_MERGE_TIME_HUGE_PAGES;
840 break;
841 case 'D':
842 test_name = KSM_UNMERGE_TIME;
843 break;
844 case 'C':
845 test_name = KSM_COW_TIME;
846 break;
847 default:
848 return KSFT_FAIL;
849 }
850 }
851
852 if (prot == 0)
853 prot = str_to_prot(KSM_PROT_STR_DEFAULT);
854
855 if (access(KSM_SYSFS_PATH, F_OK)) {
856 printf("Config KSM not enabled\n");
857 return KSFT_SKIP;
858 }
859
860 if (ksm_save_def(&ksm_sysfs_old)) {
861 printf("Cannot save default tunables\n");
862 return KSFT_FAIL;
863 }
864
865 if (ksm_write_sysfs(KSM_FP("run"), 2) ||
866 ksm_write_sysfs(KSM_FP("sleep_millisecs"), 0) ||
867 numa_available() ? 0 :
868 ksm_write_sysfs(KSM_FP("merge_across_nodes"), 1) ||
869 ksm_write_sysfs(KSM_FP("pages_to_scan"), page_count))
870 return KSFT_FAIL;
871
872 switch (test_name) {
873 case CHECK_KSM_MERGE:
874 ret = check_ksm_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, page_count,
875 ksm_scan_limit_sec, page_size);
876 break;
877 case CHECK_KSM_UNMERGE:
878 ret = check_ksm_unmerge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,
879 ksm_scan_limit_sec, page_size);
880 break;
881 case CHECK_KSM_ZERO_PAGE_MERGE:
882 ret = check_ksm_zero_page_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,
883 page_count, ksm_scan_limit_sec, use_zero_pages,
884 page_size);
885 break;
886 case CHECK_KSM_NUMA_MERGE:
887 ret = check_ksm_numa_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,
888 ksm_scan_limit_sec, merge_across_nodes, page_size);
889 break;
890 case KSM_MERGE_TIME:
891 if (size_MB == 0) {
892 printf("Option '-s' is required.\n");
893 return KSFT_FAIL;
894 }
895 ret = ksm_merge_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,
896 ksm_scan_limit_sec, size_MB);
897 break;
898 case KSM_MERGE_TIME_HUGE_PAGES:
899 if (size_MB == 0) {
900 printf("Option '-s' is required.\n");
901 return KSFT_FAIL;
902 }
903 ret = ksm_merge_hugepages_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,
904 ksm_scan_limit_sec, size_MB);
905 break;
906 case KSM_UNMERGE_TIME:
907 if (size_MB == 0) {
908 printf("Option '-s' is required.\n");
909 return KSFT_FAIL;
910 }
911 ret = ksm_unmerge_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,
912 ksm_scan_limit_sec, size_MB);
913 break;
914 case KSM_COW_TIME:
915 ret = ksm_cow_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,
916 ksm_scan_limit_sec, page_size);
917 break;
918 }
919
920 if (ksm_restore(&ksm_sysfs_old)) {
921 printf("Cannot restore default tunables\n");
922 return KSFT_FAIL;
923 }
924
925 return ret;
926 }
927