1 // SPDX-License-Identifier: GPL-2.0
2
3 #include <sys/mman.h>
4 #include <sys/prctl.h>
5 #include <sys/wait.h>
6 #include <stdbool.h>
7 #include <time.h>
8 #include <string.h>
9 #include <numa.h>
10 #include <unistd.h>
11 #include <fcntl.h>
12 #include <stdint.h>
13 #include <err.h>
14
15 #include "../kselftest.h"
16 #include <include/vdso/time64.h>
17 #include "vm_util.h"
18
19 #define KSM_SYSFS_PATH "/sys/kernel/mm/ksm/"
20 #define KSM_FP(s) (KSM_SYSFS_PATH s)
21 #define KSM_SCAN_LIMIT_SEC_DEFAULT 120
22 #define KSM_PAGE_COUNT_DEFAULT 10l
23 #define KSM_PROT_STR_DEFAULT "rw"
24 #define KSM_USE_ZERO_PAGES_DEFAULT false
25 #define KSM_MERGE_ACROSS_NODES_DEFAULT true
26 #define KSM_MERGE_TYPE_DEFAULT 0
27 #define MB (1ul << 20)
28
29 struct ksm_sysfs {
30 unsigned long max_page_sharing;
31 unsigned long merge_across_nodes;
32 unsigned long pages_to_scan;
33 unsigned long run;
34 unsigned long sleep_millisecs;
35 unsigned long stable_node_chains_prune_millisecs;
36 unsigned long use_zero_pages;
37 };
38
39 enum ksm_merge_type {
40 KSM_MERGE_MADVISE,
41 KSM_MERGE_PRCTL,
42 KSM_MERGE_LAST = KSM_MERGE_PRCTL
43 };
44
45 enum ksm_test_name {
46 CHECK_KSM_MERGE,
47 CHECK_KSM_UNMERGE,
48 CHECK_KSM_GET_MERGE_TYPE,
49 CHECK_KSM_ZERO_PAGE_MERGE,
50 CHECK_KSM_NUMA_MERGE,
51 KSM_MERGE_TIME,
52 KSM_MERGE_TIME_HUGE_PAGES,
53 KSM_UNMERGE_TIME,
54 KSM_COW_TIME
55 };
56
57 int debug;
58
ksm_write_sysfs(const char * file_path,unsigned long val)59 static int ksm_write_sysfs(const char *file_path, unsigned long val)
60 {
61 return write_sysfs(file_path, val);
62 }
63
ksm_read_sysfs(const char * file_path,unsigned long * val)64 static int ksm_read_sysfs(const char *file_path, unsigned long *val)
65 {
66 return read_sysfs(file_path, val);
67 }
68
ksm_print_sysfs(void)69 static void ksm_print_sysfs(void)
70 {
71 unsigned long max_page_sharing, pages_sharing, pages_shared;
72 unsigned long full_scans, pages_unshared, pages_volatile;
73 unsigned long stable_node_chains, stable_node_dups;
74 long general_profit;
75
76 if (ksm_read_sysfs(KSM_FP("pages_shared"), &pages_shared) ||
77 ksm_read_sysfs(KSM_FP("pages_sharing"), &pages_sharing) ||
78 ksm_read_sysfs(KSM_FP("max_page_sharing"), &max_page_sharing) ||
79 ksm_read_sysfs(KSM_FP("full_scans"), &full_scans) ||
80 ksm_read_sysfs(KSM_FP("pages_unshared"), &pages_unshared) ||
81 ksm_read_sysfs(KSM_FP("pages_volatile"), &pages_volatile) ||
82 ksm_read_sysfs(KSM_FP("stable_node_chains"), &stable_node_chains) ||
83 ksm_read_sysfs(KSM_FP("stable_node_dups"), &stable_node_dups) ||
84 ksm_read_sysfs(KSM_FP("general_profit"), (unsigned long *)&general_profit))
85 return;
86
87 printf("pages_shared : %lu\n", pages_shared);
88 printf("pages_sharing : %lu\n", pages_sharing);
89 printf("max_page_sharing : %lu\n", max_page_sharing);
90 printf("full_scans : %lu\n", full_scans);
91 printf("pages_unshared : %lu\n", pages_unshared);
92 printf("pages_volatile : %lu\n", pages_volatile);
93 printf("stable_node_chains: %lu\n", stable_node_chains);
94 printf("stable_node_dups : %lu\n", stable_node_dups);
95 printf("general_profit : %ld\n", general_profit);
96 }
97
ksm_print_procfs(void)98 static void ksm_print_procfs(void)
99 {
100 const char *file_name = "/proc/self/ksm_stat";
101 char buffer[512];
102 FILE *f = fopen(file_name, "r");
103
104 if (!f) {
105 fprintf(stderr, "f %s\n", file_name);
106 perror("fopen");
107 return;
108 }
109
110 while (fgets(buffer, sizeof(buffer), f))
111 printf("%s", buffer);
112
113 fclose(f);
114 }
115
str_to_prot(char * prot_str)116 static int str_to_prot(char *prot_str)
117 {
118 int prot = 0;
119
120 if ((strchr(prot_str, 'r')) != NULL)
121 prot |= PROT_READ;
122 if ((strchr(prot_str, 'w')) != NULL)
123 prot |= PROT_WRITE;
124 if ((strchr(prot_str, 'x')) != NULL)
125 prot |= PROT_EXEC;
126
127 return prot;
128 }
129
print_help(void)130 static void print_help(void)
131 {
132 printf("usage: ksm_tests [-h] <test type> [-a prot] [-p page_count] [-l timeout]\n"
133 "[-z use_zero_pages] [-m merge_across_nodes] [-s size]\n");
134
135 printf("Supported <test type>:\n"
136 " -M (page merging)\n"
137 " -Z (zero pages merging)\n"
138 " -N (merging of pages in different NUMA nodes)\n"
139 " -U (page unmerging)\n"
140 " -P evaluate merging time and speed.\n"
141 " For this test, the size of duplicated memory area (in MiB)\n"
142 " must be provided using -s option\n"
143 " -H evaluate merging time and speed of area allocated mostly with huge pages\n"
144 " For this test, the size of duplicated memory area (in MiB)\n"
145 " must be provided using -s option\n"
146 " -D evaluate unmerging time and speed when disabling KSM.\n"
147 " For this test, the size of duplicated memory area (in MiB)\n"
148 " must be provided using -s option\n"
149 " -C evaluate the time required to break COW of merged pages.\n\n");
150
151 printf(" -a: specify the access protections of pages.\n"
152 " <prot> must be of the form [rwx].\n"
153 " Default: %s\n", KSM_PROT_STR_DEFAULT);
154 printf(" -p: specify the number of pages to test.\n"
155 " Default: %ld\n", KSM_PAGE_COUNT_DEFAULT);
156 printf(" -l: limit the maximum running time (in seconds) for a test.\n"
157 " Default: %d seconds\n", KSM_SCAN_LIMIT_SEC_DEFAULT);
158 printf(" -z: change use_zero_pages tunable\n"
159 " Default: %d\n", KSM_USE_ZERO_PAGES_DEFAULT);
160 printf(" -m: change merge_across_nodes tunable\n"
161 " Default: %d\n", KSM_MERGE_ACROSS_NODES_DEFAULT);
162 printf(" -d: turn debugging output on\n");
163 printf(" -s: the size of duplicated memory area (in MiB)\n");
164 printf(" -t: KSM merge type\n"
165 " Default: 0\n"
166 " 0: madvise merging\n"
167 " 1: prctl merging\n");
168
169 exit(0);
170 }
171
allocate_memory(void * ptr,int prot,int mapping,char data,size_t map_size)172 static void *allocate_memory(void *ptr, int prot, int mapping, char data, size_t map_size)
173 {
174 void *map_ptr = mmap(ptr, map_size, PROT_WRITE, mapping, -1, 0);
175
176 if (!map_ptr) {
177 perror("mmap");
178 return NULL;
179 }
180 memset(map_ptr, data, map_size);
181 if (mprotect(map_ptr, map_size, prot)) {
182 perror("mprotect");
183 munmap(map_ptr, map_size);
184 return NULL;
185 }
186
187 return map_ptr;
188 }
189
ksm_do_scan(int scan_count,struct timespec start_time,int timeout)190 static int ksm_do_scan(int scan_count, struct timespec start_time, int timeout)
191 {
192 struct timespec cur_time;
193 unsigned long cur_scan, init_scan;
194
195 if (ksm_read_sysfs(KSM_FP("full_scans"), &init_scan))
196 return 1;
197 cur_scan = init_scan;
198
199 while (cur_scan < init_scan + scan_count) {
200 if (ksm_read_sysfs(KSM_FP("full_scans"), &cur_scan))
201 return 1;
202 if (clock_gettime(CLOCK_MONOTONIC_RAW, &cur_time)) {
203 perror("clock_gettime");
204 return 1;
205 }
206 if ((cur_time.tv_sec - start_time.tv_sec) > timeout) {
207 printf("Scan time limit exceeded\n");
208 return 1;
209 }
210 }
211
212 return 0;
213 }
214
ksm_merge_pages(int merge_type,void * addr,size_t size,struct timespec start_time,int timeout)215 static int ksm_merge_pages(int merge_type, void *addr, size_t size,
216 struct timespec start_time, int timeout)
217 {
218 if (merge_type == KSM_MERGE_MADVISE) {
219 if (madvise(addr, size, MADV_MERGEABLE)) {
220 perror("madvise");
221 return 1;
222 }
223 } else if (merge_type == KSM_MERGE_PRCTL) {
224 if (prctl(PR_SET_MEMORY_MERGE, 1, 0, 0, 0)) {
225 perror("prctl");
226 return 1;
227 }
228 }
229
230 if (ksm_write_sysfs(KSM_FP("run"), 1))
231 return 1;
232
233 /* Since merging occurs only after 2 scans, make sure to get at least 2 full scans */
234 if (ksm_do_scan(2, start_time, timeout))
235 return 1;
236
237 return 0;
238 }
239
ksm_unmerge_pages(void * addr,size_t size,struct timespec start_time,int timeout)240 static int ksm_unmerge_pages(void *addr, size_t size,
241 struct timespec start_time, int timeout)
242 {
243 if (madvise(addr, size, MADV_UNMERGEABLE)) {
244 perror("madvise");
245 return 1;
246 }
247 return 0;
248 }
249
assert_ksm_pages_count(long dupl_page_count)250 static bool assert_ksm_pages_count(long dupl_page_count)
251 {
252 unsigned long max_page_sharing, pages_sharing, pages_shared;
253
254 if (ksm_read_sysfs(KSM_FP("pages_shared"), &pages_shared) ||
255 ksm_read_sysfs(KSM_FP("pages_sharing"), &pages_sharing) ||
256 ksm_read_sysfs(KSM_FP("max_page_sharing"), &max_page_sharing))
257 return false;
258
259 if (debug) {
260 ksm_print_sysfs();
261 ksm_print_procfs();
262 }
263
264 /*
265 * Since there must be at least 2 pages for merging and 1 page can be
266 * shared with the limited number of pages (max_page_sharing), sometimes
267 * there are 'leftover' pages that cannot be merged. For example, if there
268 * are 11 pages and max_page_sharing = 10, then only 10 pages will be
269 * merged and the 11th page won't be affected. As a result, when the number
270 * of duplicate pages is divided by max_page_sharing and the remainder is 1,
271 * pages_shared and pages_sharing values will be equal between dupl_page_count
272 * and dupl_page_count - 1.
273 */
274 if (dupl_page_count % max_page_sharing == 1 || dupl_page_count % max_page_sharing == 0) {
275 if (pages_shared == dupl_page_count / max_page_sharing &&
276 pages_sharing == pages_shared * (max_page_sharing - 1))
277 return true;
278 } else {
279 if (pages_shared == (dupl_page_count / max_page_sharing + 1) &&
280 pages_sharing == dupl_page_count - pages_shared)
281 return true;
282 }
283
284 return false;
285 }
286
ksm_save_def(struct ksm_sysfs * ksm_sysfs)287 static int ksm_save_def(struct ksm_sysfs *ksm_sysfs)
288 {
289 if (ksm_read_sysfs(KSM_FP("max_page_sharing"), &ksm_sysfs->max_page_sharing) ||
290 numa_available() ? 0 :
291 ksm_read_sysfs(KSM_FP("merge_across_nodes"), &ksm_sysfs->merge_across_nodes) ||
292 ksm_read_sysfs(KSM_FP("sleep_millisecs"), &ksm_sysfs->sleep_millisecs) ||
293 ksm_read_sysfs(KSM_FP("pages_to_scan"), &ksm_sysfs->pages_to_scan) ||
294 ksm_read_sysfs(KSM_FP("run"), &ksm_sysfs->run) ||
295 ksm_read_sysfs(KSM_FP("stable_node_chains_prune_millisecs"),
296 &ksm_sysfs->stable_node_chains_prune_millisecs) ||
297 ksm_read_sysfs(KSM_FP("use_zero_pages"), &ksm_sysfs->use_zero_pages))
298 return 1;
299
300 return 0;
301 }
302
ksm_restore(struct ksm_sysfs * ksm_sysfs)303 static int ksm_restore(struct ksm_sysfs *ksm_sysfs)
304 {
305 if (ksm_write_sysfs(KSM_FP("max_page_sharing"), ksm_sysfs->max_page_sharing) ||
306 numa_available() ? 0 :
307 ksm_write_sysfs(KSM_FP("merge_across_nodes"), ksm_sysfs->merge_across_nodes) ||
308 ksm_write_sysfs(KSM_FP("pages_to_scan"), ksm_sysfs->pages_to_scan) ||
309 ksm_write_sysfs(KSM_FP("run"), ksm_sysfs->run) ||
310 ksm_write_sysfs(KSM_FP("sleep_millisecs"), ksm_sysfs->sleep_millisecs) ||
311 ksm_write_sysfs(KSM_FP("stable_node_chains_prune_millisecs"),
312 ksm_sysfs->stable_node_chains_prune_millisecs) ||
313 ksm_write_sysfs(KSM_FP("use_zero_pages"), ksm_sysfs->use_zero_pages))
314 return 1;
315
316 return 0;
317 }
318
check_ksm_merge(int merge_type,int mapping,int prot,long page_count,int timeout,size_t page_size)319 static int check_ksm_merge(int merge_type, int mapping, int prot,
320 long page_count, int timeout, size_t page_size)
321 {
322 void *map_ptr;
323 struct timespec start_time;
324
325 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
326 perror("clock_gettime");
327 return KSFT_FAIL;
328 }
329
330 /* fill pages with the same data and merge them */
331 map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count);
332 if (!map_ptr)
333 return KSFT_FAIL;
334
335 if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout))
336 goto err_out;
337
338 /* verify that the right number of pages are merged */
339 if (assert_ksm_pages_count(page_count)) {
340 printf("OK\n");
341 munmap(map_ptr, page_size * page_count);
342 if (merge_type == KSM_MERGE_PRCTL)
343 prctl(PR_SET_MEMORY_MERGE, 0, 0, 0, 0);
344 return KSFT_PASS;
345 }
346
347 err_out:
348 printf("Not OK\n");
349 munmap(map_ptr, page_size * page_count);
350 return KSFT_FAIL;
351 }
352
check_ksm_unmerge(int merge_type,int mapping,int prot,int timeout,size_t page_size)353 static int check_ksm_unmerge(int merge_type, int mapping, int prot, int timeout, size_t page_size)
354 {
355 void *map_ptr;
356 struct timespec start_time;
357 int page_count = 2;
358
359 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
360 perror("clock_gettime");
361 return KSFT_FAIL;
362 }
363
364 /* fill pages with the same data and merge them */
365 map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count);
366 if (!map_ptr)
367 return KSFT_FAIL;
368
369 if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout))
370 goto err_out;
371
372 /* change 1 byte in each of the 2 pages -- KSM must automatically unmerge them */
373 memset(map_ptr, '-', 1);
374 memset(map_ptr + page_size, '+', 1);
375
376 /* get at least 1 scan, so KSM can detect that the pages were modified */
377 if (ksm_do_scan(1, start_time, timeout))
378 goto err_out;
379
380 /* check that unmerging was successful and 0 pages are currently merged */
381 if (assert_ksm_pages_count(0)) {
382 printf("OK\n");
383 munmap(map_ptr, page_size * page_count);
384 return KSFT_PASS;
385 }
386
387 err_out:
388 printf("Not OK\n");
389 munmap(map_ptr, page_size * page_count);
390 return KSFT_FAIL;
391 }
392
check_ksm_zero_page_merge(int merge_type,int mapping,int prot,long page_count,int timeout,bool use_zero_pages,size_t page_size)393 static int check_ksm_zero_page_merge(int merge_type, int mapping, int prot, long page_count,
394 int timeout, bool use_zero_pages, size_t page_size)
395 {
396 void *map_ptr;
397 struct timespec start_time;
398
399 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
400 perror("clock_gettime");
401 return KSFT_FAIL;
402 }
403
404 if (ksm_write_sysfs(KSM_FP("use_zero_pages"), use_zero_pages))
405 return KSFT_FAIL;
406
407 /* fill pages with zero and try to merge them */
408 map_ptr = allocate_memory(NULL, prot, mapping, 0, page_size * page_count);
409 if (!map_ptr)
410 return KSFT_FAIL;
411
412 if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout))
413 goto err_out;
414
415 /*
416 * verify that the right number of pages are merged:
417 * 1) if use_zero_pages is set to 1, empty pages are merged
418 * with the kernel zero page instead of with each other;
419 * 2) if use_zero_pages is set to 0, empty pages are not treated specially
420 * and merged as usual.
421 */
422 if (use_zero_pages && !assert_ksm_pages_count(0))
423 goto err_out;
424 else if (!use_zero_pages && !assert_ksm_pages_count(page_count))
425 goto err_out;
426
427 printf("OK\n");
428 munmap(map_ptr, page_size * page_count);
429 return KSFT_PASS;
430
431 err_out:
432 printf("Not OK\n");
433 munmap(map_ptr, page_size * page_count);
434 return KSFT_FAIL;
435 }
436
get_next_mem_node(int node)437 static int get_next_mem_node(int node)
438 {
439
440 long node_size;
441 int mem_node = 0;
442 int i, max_node = numa_max_node();
443
444 for (i = node + 1; i <= max_node + node; i++) {
445 mem_node = i % (max_node + 1);
446 node_size = numa_node_size(mem_node, NULL);
447 if (node_size > 0)
448 break;
449 }
450 return mem_node;
451 }
452
get_first_mem_node(void)453 static int get_first_mem_node(void)
454 {
455 return get_next_mem_node(numa_max_node());
456 }
457
check_ksm_numa_merge(int merge_type,int mapping,int prot,int timeout,bool merge_across_nodes,size_t page_size)458 static int check_ksm_numa_merge(int merge_type, int mapping, int prot, int timeout,
459 bool merge_across_nodes, size_t page_size)
460 {
461 void *numa1_map_ptr, *numa2_map_ptr;
462 struct timespec start_time;
463 int page_count = 2;
464 int first_node;
465
466 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
467 perror("clock_gettime");
468 return KSFT_FAIL;
469 }
470
471 if (numa_available() < 0) {
472 perror("NUMA support not enabled");
473 return KSFT_SKIP;
474 }
475 if (numa_num_configured_nodes() <= 1) {
476 printf("At least 2 NUMA nodes must be available\n");
477 return KSFT_SKIP;
478 }
479 if (ksm_write_sysfs(KSM_FP("merge_across_nodes"), merge_across_nodes))
480 return KSFT_FAIL;
481
482 /* allocate 2 pages in 2 different NUMA nodes and fill them with the same data */
483 first_node = get_first_mem_node();
484 numa1_map_ptr = numa_alloc_onnode(page_size, first_node);
485 numa2_map_ptr = numa_alloc_onnode(page_size, get_next_mem_node(first_node));
486 if (!numa1_map_ptr || !numa2_map_ptr) {
487 perror("numa_alloc_onnode");
488 return KSFT_FAIL;
489 }
490
491 memset(numa1_map_ptr, '*', page_size);
492 memset(numa2_map_ptr, '*', page_size);
493
494 /* try to merge the pages */
495 if (ksm_merge_pages(merge_type, numa1_map_ptr, page_size, start_time, timeout) ||
496 ksm_merge_pages(merge_type, numa2_map_ptr, page_size, start_time, timeout))
497 goto err_out;
498
499 /*
500 * verify that the right number of pages are merged:
501 * 1) if merge_across_nodes was enabled, 2 duplicate pages will be merged;
502 * 2) if merge_across_nodes = 0, there must be 0 merged pages, since there is
503 * only 1 unique page in each node and they can't be shared.
504 */
505 if (merge_across_nodes && !assert_ksm_pages_count(page_count))
506 goto err_out;
507 else if (!merge_across_nodes && !assert_ksm_pages_count(0))
508 goto err_out;
509
510 numa_free(numa1_map_ptr, page_size);
511 numa_free(numa2_map_ptr, page_size);
512 printf("OK\n");
513 return KSFT_PASS;
514
515 err_out:
516 numa_free(numa1_map_ptr, page_size);
517 numa_free(numa2_map_ptr, page_size);
518 printf("Not OK\n");
519 return KSFT_FAIL;
520 }
521
ksm_merge_hugepages_time(int merge_type,int mapping,int prot,int timeout,size_t map_size)522 static int ksm_merge_hugepages_time(int merge_type, int mapping, int prot,
523 int timeout, size_t map_size)
524 {
525 void *map_ptr, *map_ptr_orig;
526 struct timespec start_time, end_time;
527 unsigned long scan_time_ns;
528 int pagemap_fd, n_normal_pages, n_huge_pages;
529
530 map_size *= MB;
531 size_t len = map_size;
532
533 len -= len % HPAGE_SIZE;
534 map_ptr_orig = mmap(NULL, len + HPAGE_SIZE, PROT_READ | PROT_WRITE,
535 MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE, -1, 0);
536 map_ptr = map_ptr_orig + HPAGE_SIZE - (uintptr_t)map_ptr_orig % HPAGE_SIZE;
537
538 if (map_ptr_orig == MAP_FAILED)
539 err(2, "initial mmap");
540
541 if (madvise(map_ptr, len, MADV_HUGEPAGE))
542 err(2, "MADV_HUGEPAGE");
543
544 pagemap_fd = open("/proc/self/pagemap", O_RDONLY);
545 if (pagemap_fd < 0)
546 err(2, "open pagemap");
547
548 n_normal_pages = 0;
549 n_huge_pages = 0;
550 for (void *p = map_ptr; p < map_ptr + len; p += HPAGE_SIZE) {
551 if (allocate_transhuge(p, pagemap_fd) < 0)
552 n_normal_pages++;
553 else
554 n_huge_pages++;
555 }
556 printf("Number of normal pages: %d\n", n_normal_pages);
557 printf("Number of huge pages: %d\n", n_huge_pages);
558
559 memset(map_ptr, '*', len);
560
561 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
562 perror("clock_gettime");
563 goto err_out;
564 }
565 if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout))
566 goto err_out;
567 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) {
568 perror("clock_gettime");
569 goto err_out;
570 }
571
572 scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC +
573 (end_time.tv_nsec - start_time.tv_nsec);
574
575 printf("Total size: %lu MiB\n", map_size / MB);
576 printf("Total time: %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC,
577 scan_time_ns % NSEC_PER_SEC);
578 printf("Average speed: %.3f MiB/s\n", (map_size / MB) /
579 ((double)scan_time_ns / NSEC_PER_SEC));
580
581 munmap(map_ptr_orig, len + HPAGE_SIZE);
582 return KSFT_PASS;
583
584 err_out:
585 printf("Not OK\n");
586 munmap(map_ptr_orig, len + HPAGE_SIZE);
587 return KSFT_FAIL;
588 }
589
ksm_merge_time(int merge_type,int mapping,int prot,int timeout,size_t map_size)590 static int ksm_merge_time(int merge_type, int mapping, int prot, int timeout, size_t map_size)
591 {
592 void *map_ptr;
593 struct timespec start_time, end_time;
594 unsigned long scan_time_ns;
595
596 map_size *= MB;
597
598 map_ptr = allocate_memory(NULL, prot, mapping, '*', map_size);
599 if (!map_ptr)
600 return KSFT_FAIL;
601
602 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
603 perror("clock_gettime");
604 goto err_out;
605 }
606 if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout))
607 goto err_out;
608 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) {
609 perror("clock_gettime");
610 goto err_out;
611 }
612
613 scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC +
614 (end_time.tv_nsec - start_time.tv_nsec);
615
616 printf("Total size: %lu MiB\n", map_size / MB);
617 printf("Total time: %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC,
618 scan_time_ns % NSEC_PER_SEC);
619 printf("Average speed: %.3f MiB/s\n", (map_size / MB) /
620 ((double)scan_time_ns / NSEC_PER_SEC));
621
622 munmap(map_ptr, map_size);
623 return KSFT_PASS;
624
625 err_out:
626 printf("Not OK\n");
627 munmap(map_ptr, map_size);
628 return KSFT_FAIL;
629 }
630
ksm_unmerge_time(int merge_type,int mapping,int prot,int timeout,size_t map_size)631 static int ksm_unmerge_time(int merge_type, int mapping, int prot, int timeout, size_t map_size)
632 {
633 void *map_ptr;
634 struct timespec start_time, end_time;
635 unsigned long scan_time_ns;
636
637 map_size *= MB;
638
639 map_ptr = allocate_memory(NULL, prot, mapping, '*', map_size);
640 if (!map_ptr)
641 return KSFT_FAIL;
642 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
643 perror("clock_gettime");
644 goto err_out;
645 }
646 if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout))
647 goto err_out;
648
649 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
650 perror("clock_gettime");
651 goto err_out;
652 }
653 if (ksm_unmerge_pages(map_ptr, map_size, start_time, timeout))
654 goto err_out;
655 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) {
656 perror("clock_gettime");
657 goto err_out;
658 }
659
660 scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC +
661 (end_time.tv_nsec - start_time.tv_nsec);
662
663 printf("Total size: %lu MiB\n", map_size / MB);
664 printf("Total time: %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC,
665 scan_time_ns % NSEC_PER_SEC);
666 printf("Average speed: %.3f MiB/s\n", (map_size / MB) /
667 ((double)scan_time_ns / NSEC_PER_SEC));
668
669 munmap(map_ptr, map_size);
670 return KSFT_PASS;
671
672 err_out:
673 printf("Not OK\n");
674 munmap(map_ptr, map_size);
675 return KSFT_FAIL;
676 }
677
ksm_cow_time(int merge_type,int mapping,int prot,int timeout,size_t page_size)678 static int ksm_cow_time(int merge_type, int mapping, int prot, int timeout, size_t page_size)
679 {
680 void *map_ptr;
681 struct timespec start_time, end_time;
682 unsigned long cow_time_ns;
683
684 /* page_count must be less than 2*page_size */
685 size_t page_count = 4000;
686
687 map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count);
688 if (!map_ptr)
689 return KSFT_FAIL;
690
691 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
692 perror("clock_gettime");
693 return KSFT_FAIL;
694 }
695 for (size_t i = 0; i < page_count - 1; i = i + 2)
696 memset(map_ptr + page_size * i, '-', 1);
697 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) {
698 perror("clock_gettime");
699 return KSFT_FAIL;
700 }
701
702 cow_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC +
703 (end_time.tv_nsec - start_time.tv_nsec);
704
705 printf("Total size: %lu MiB\n\n", (page_size * page_count) / MB);
706 printf("Not merged pages:\n");
707 printf("Total time: %ld.%09ld s\n", cow_time_ns / NSEC_PER_SEC,
708 cow_time_ns % NSEC_PER_SEC);
709 printf("Average speed: %.3f MiB/s\n\n", ((page_size * (page_count / 2)) / MB) /
710 ((double)cow_time_ns / NSEC_PER_SEC));
711
712 /* Create 2000 pairs of duplicate pages */
713 for (size_t i = 0; i < page_count - 1; i = i + 2) {
714 memset(map_ptr + page_size * i, '+', i / 2 + 1);
715 memset(map_ptr + page_size * (i + 1), '+', i / 2 + 1);
716 }
717 if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout))
718 goto err_out;
719
720 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
721 perror("clock_gettime");
722 goto err_out;
723 }
724 for (size_t i = 0; i < page_count - 1; i = i + 2)
725 memset(map_ptr + page_size * i, '-', 1);
726 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) {
727 perror("clock_gettime");
728 goto err_out;
729 }
730
731 cow_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC +
732 (end_time.tv_nsec - start_time.tv_nsec);
733
734 printf("Merged pages:\n");
735 printf("Total time: %ld.%09ld s\n", cow_time_ns / NSEC_PER_SEC,
736 cow_time_ns % NSEC_PER_SEC);
737 printf("Average speed: %.3f MiB/s\n", ((page_size * (page_count / 2)) / MB) /
738 ((double)cow_time_ns / NSEC_PER_SEC));
739
740 munmap(map_ptr, page_size * page_count);
741 return KSFT_PASS;
742
743 err_out:
744 printf("Not OK\n");
745 munmap(map_ptr, page_size * page_count);
746 return KSFT_FAIL;
747 }
748
main(int argc,char * argv[])749 int main(int argc, char *argv[])
750 {
751 int ret = 0, opt;
752 int prot = 0;
753 int ksm_scan_limit_sec = KSM_SCAN_LIMIT_SEC_DEFAULT;
754 int merge_type = KSM_MERGE_TYPE_DEFAULT;
755 long page_count = KSM_PAGE_COUNT_DEFAULT;
756 size_t page_size = sysconf(_SC_PAGESIZE);
757 struct ksm_sysfs ksm_sysfs_old;
758 int test_name = CHECK_KSM_MERGE;
759 bool use_zero_pages = KSM_USE_ZERO_PAGES_DEFAULT;
760 bool merge_across_nodes = KSM_MERGE_ACROSS_NODES_DEFAULT;
761 long size_MB = 0;
762
763 while ((opt = getopt(argc, argv, "dha:p:l:z:m:s:t:MUZNPCHD")) != -1) {
764 switch (opt) {
765 case 'a':
766 prot = str_to_prot(optarg);
767 break;
768 case 'p':
769 page_count = atol(optarg);
770 if (page_count <= 0) {
771 printf("The number of pages must be greater than 0\n");
772 return KSFT_FAIL;
773 }
774 break;
775 case 'l':
776 ksm_scan_limit_sec = atoi(optarg);
777 if (ksm_scan_limit_sec <= 0) {
778 printf("Timeout value must be greater than 0\n");
779 return KSFT_FAIL;
780 }
781 break;
782 case 'h':
783 print_help();
784 break;
785 case 'z':
786 if (strcmp(optarg, "0") == 0)
787 use_zero_pages = 0;
788 else
789 use_zero_pages = 1;
790 break;
791 case 'm':
792 if (strcmp(optarg, "0") == 0)
793 merge_across_nodes = 0;
794 else
795 merge_across_nodes = 1;
796 break;
797 case 'd':
798 debug = 1;
799 break;
800 case 's':
801 size_MB = atoi(optarg);
802 if (size_MB <= 0) {
803 printf("Size must be greater than 0\n");
804 return KSFT_FAIL;
805 }
806 break;
807 case 't':
808 {
809 int tmp = atoi(optarg);
810
811 if (tmp < 0 || tmp > KSM_MERGE_LAST) {
812 printf("Invalid merge type\n");
813 return KSFT_FAIL;
814 }
815 merge_type = tmp;
816 }
817 break;
818 case 'M':
819 break;
820 case 'U':
821 test_name = CHECK_KSM_UNMERGE;
822 break;
823 case 'Z':
824 test_name = CHECK_KSM_ZERO_PAGE_MERGE;
825 break;
826 case 'N':
827 test_name = CHECK_KSM_NUMA_MERGE;
828 break;
829 case 'P':
830 test_name = KSM_MERGE_TIME;
831 break;
832 case 'H':
833 test_name = KSM_MERGE_TIME_HUGE_PAGES;
834 break;
835 case 'D':
836 test_name = KSM_UNMERGE_TIME;
837 break;
838 case 'C':
839 test_name = KSM_COW_TIME;
840 break;
841 default:
842 return KSFT_FAIL;
843 }
844 }
845
846 if (prot == 0)
847 prot = str_to_prot(KSM_PROT_STR_DEFAULT);
848
849 if (access(KSM_SYSFS_PATH, F_OK)) {
850 printf("Config KSM not enabled\n");
851 return KSFT_SKIP;
852 }
853
854 if (ksm_save_def(&ksm_sysfs_old)) {
855 printf("Cannot save default tunables\n");
856 return KSFT_FAIL;
857 }
858
859 if (ksm_write_sysfs(KSM_FP("run"), 2) ||
860 ksm_write_sysfs(KSM_FP("sleep_millisecs"), 0) ||
861 numa_available() ? 0 :
862 ksm_write_sysfs(KSM_FP("merge_across_nodes"), 1) ||
863 ksm_write_sysfs(KSM_FP("pages_to_scan"), page_count))
864 return KSFT_FAIL;
865
866 switch (test_name) {
867 case CHECK_KSM_MERGE:
868 ret = check_ksm_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, page_count,
869 ksm_scan_limit_sec, page_size);
870 break;
871 case CHECK_KSM_UNMERGE:
872 ret = check_ksm_unmerge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,
873 ksm_scan_limit_sec, page_size);
874 break;
875 case CHECK_KSM_ZERO_PAGE_MERGE:
876 ret = check_ksm_zero_page_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,
877 page_count, ksm_scan_limit_sec, use_zero_pages,
878 page_size);
879 break;
880 case CHECK_KSM_NUMA_MERGE:
881 ret = check_ksm_numa_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,
882 ksm_scan_limit_sec, merge_across_nodes, page_size);
883 break;
884 case KSM_MERGE_TIME:
885 if (size_MB == 0) {
886 printf("Option '-s' is required.\n");
887 return KSFT_FAIL;
888 }
889 ret = ksm_merge_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,
890 ksm_scan_limit_sec, size_MB);
891 break;
892 case KSM_MERGE_TIME_HUGE_PAGES:
893 if (size_MB == 0) {
894 printf("Option '-s' is required.\n");
895 return KSFT_FAIL;
896 }
897 ret = ksm_merge_hugepages_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,
898 ksm_scan_limit_sec, size_MB);
899 break;
900 case KSM_UNMERGE_TIME:
901 if (size_MB == 0) {
902 printf("Option '-s' is required.\n");
903 return KSFT_FAIL;
904 }
905 ret = ksm_unmerge_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,
906 ksm_scan_limit_sec, size_MB);
907 break;
908 case KSM_COW_TIME:
909 ret = ksm_cow_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,
910 ksm_scan_limit_sec, page_size);
911 break;
912 }
913
914 if (ksm_restore(&ksm_sysfs_old)) {
915 printf("Cannot restore default tunables\n");
916 return KSFT_FAIL;
917 }
918
919 return ret;
920 }
921