xref: /linux/tools/testing/selftests/mm/ksm_tests.c (revision e9ef810dfee7a2227da9d423aecb0ced35faddbe)
1 // SPDX-License-Identifier: GPL-2.0
2 
3 #include <sys/mman.h>
4 #include <sys/prctl.h>
5 #include <sys/wait.h>
6 #include <stdbool.h>
7 #include <time.h>
8 #include <string.h>
9 #include <numa.h>
10 #include <unistd.h>
11 #include <fcntl.h>
12 #include <stdint.h>
13 #include <err.h>
14 
15 #include "../kselftest.h"
16 #include <include/vdso/time64.h>
17 #include "vm_util.h"
18 #include "thp_settings.h"
19 
20 #define KSM_SYSFS_PATH "/sys/kernel/mm/ksm/"
21 #define KSM_FP(s) (KSM_SYSFS_PATH s)
22 #define KSM_SCAN_LIMIT_SEC_DEFAULT 120
23 #define KSM_PAGE_COUNT_DEFAULT 10l
24 #define KSM_PROT_STR_DEFAULT "rw"
25 #define KSM_USE_ZERO_PAGES_DEFAULT false
26 #define KSM_MERGE_ACROSS_NODES_DEFAULT true
27 #define KSM_MERGE_TYPE_DEFAULT 0
28 #define MB (1ul << 20)
29 
30 struct ksm_sysfs {
31 	unsigned long max_page_sharing;
32 	unsigned long merge_across_nodes;
33 	unsigned long pages_to_scan;
34 	unsigned long run;
35 	unsigned long sleep_millisecs;
36 	unsigned long stable_node_chains_prune_millisecs;
37 	unsigned long use_zero_pages;
38 };
39 
40 enum ksm_merge_type {
41 	KSM_MERGE_MADVISE,
42 	KSM_MERGE_PRCTL,
43 	KSM_MERGE_LAST = KSM_MERGE_PRCTL
44 };
45 
46 enum ksm_test_name {
47 	CHECK_KSM_MERGE,
48 	CHECK_KSM_UNMERGE,
49 	CHECK_KSM_GET_MERGE_TYPE,
50 	CHECK_KSM_ZERO_PAGE_MERGE,
51 	CHECK_KSM_NUMA_MERGE,
52 	KSM_MERGE_TIME,
53 	KSM_MERGE_TIME_HUGE_PAGES,
54 	KSM_UNMERGE_TIME,
55 	KSM_COW_TIME
56 };
57 
58 int debug;
59 
ksm_write_sysfs(const char * file_path,unsigned long val)60 static int ksm_write_sysfs(const char *file_path, unsigned long val)
61 {
62 	return write_sysfs(file_path, val);
63 }
64 
ksm_read_sysfs(const char * file_path,unsigned long * val)65 static int ksm_read_sysfs(const char *file_path, unsigned long *val)
66 {
67 	return read_sysfs(file_path, val);
68 }
69 
ksm_print_sysfs(void)70 static void ksm_print_sysfs(void)
71 {
72 	unsigned long max_page_sharing, pages_sharing, pages_shared;
73 	unsigned long full_scans, pages_unshared, pages_volatile;
74 	unsigned long stable_node_chains, stable_node_dups;
75 	long general_profit;
76 
77 	if (ksm_read_sysfs(KSM_FP("pages_shared"), &pages_shared) ||
78 	    ksm_read_sysfs(KSM_FP("pages_sharing"), &pages_sharing) ||
79 	    ksm_read_sysfs(KSM_FP("max_page_sharing"), &max_page_sharing) ||
80 	    ksm_read_sysfs(KSM_FP("full_scans"), &full_scans) ||
81 	    ksm_read_sysfs(KSM_FP("pages_unshared"), &pages_unshared) ||
82 	    ksm_read_sysfs(KSM_FP("pages_volatile"), &pages_volatile) ||
83 	    ksm_read_sysfs(KSM_FP("stable_node_chains"), &stable_node_chains) ||
84 	    ksm_read_sysfs(KSM_FP("stable_node_dups"), &stable_node_dups) ||
85 	    ksm_read_sysfs(KSM_FP("general_profit"), (unsigned long *)&general_profit))
86 		return;
87 
88 	printf("pages_shared      : %lu\n", pages_shared);
89 	printf("pages_sharing     : %lu\n", pages_sharing);
90 	printf("max_page_sharing  : %lu\n", max_page_sharing);
91 	printf("full_scans        : %lu\n", full_scans);
92 	printf("pages_unshared    : %lu\n", pages_unshared);
93 	printf("pages_volatile    : %lu\n", pages_volatile);
94 	printf("stable_node_chains: %lu\n", stable_node_chains);
95 	printf("stable_node_dups  : %lu\n", stable_node_dups);
96 	printf("general_profit    : %ld\n", general_profit);
97 }
98 
ksm_print_procfs(void)99 static void ksm_print_procfs(void)
100 {
101 	const char *file_name = "/proc/self/ksm_stat";
102 	char buffer[512];
103 	FILE *f = fopen(file_name, "r");
104 
105 	if (!f) {
106 		fprintf(stderr, "f %s\n", file_name);
107 		perror("fopen");
108 		return;
109 	}
110 
111 	while (fgets(buffer, sizeof(buffer), f))
112 		printf("%s", buffer);
113 
114 	fclose(f);
115 }
116 
str_to_prot(char * prot_str)117 static int str_to_prot(char *prot_str)
118 {
119 	int prot = 0;
120 
121 	if ((strchr(prot_str, 'r')) != NULL)
122 		prot |= PROT_READ;
123 	if ((strchr(prot_str, 'w')) != NULL)
124 		prot |= PROT_WRITE;
125 	if ((strchr(prot_str, 'x')) != NULL)
126 		prot |= PROT_EXEC;
127 
128 	return prot;
129 }
130 
print_help(void)131 static void print_help(void)
132 {
133 	printf("usage: ksm_tests [-h] <test type> [-a prot] [-p page_count] [-l timeout]\n"
134 	       "[-z use_zero_pages] [-m merge_across_nodes] [-s size]\n");
135 
136 	printf("Supported <test type>:\n"
137 	       " -M (page merging)\n"
138 	       " -Z (zero pages merging)\n"
139 	       " -N (merging of pages in different NUMA nodes)\n"
140 	       " -U (page unmerging)\n"
141 	       " -P evaluate merging time and speed.\n"
142 	       "    For this test, the size of duplicated memory area (in MiB)\n"
143 	       "    must be provided using -s option\n"
144 	       " -H evaluate merging time and speed of area allocated mostly with huge pages\n"
145 	       "    For this test, the size of duplicated memory area (in MiB)\n"
146 	       "    must be provided using -s option\n"
147 	       " -D evaluate unmerging time and speed when disabling KSM.\n"
148 	       "    For this test, the size of duplicated memory area (in MiB)\n"
149 	       "    must be provided using -s option\n"
150 	       " -C evaluate the time required to break COW of merged pages.\n\n");
151 
152 	printf(" -a: specify the access protections of pages.\n"
153 	       "     <prot> must be of the form [rwx].\n"
154 	       "     Default: %s\n", KSM_PROT_STR_DEFAULT);
155 	printf(" -p: specify the number of pages to test.\n"
156 	       "     Default: %ld\n", KSM_PAGE_COUNT_DEFAULT);
157 	printf(" -l: limit the maximum running time (in seconds) for a test.\n"
158 	       "     Default: %d seconds\n", KSM_SCAN_LIMIT_SEC_DEFAULT);
159 	printf(" -z: change use_zero_pages tunable\n"
160 	       "     Default: %d\n", KSM_USE_ZERO_PAGES_DEFAULT);
161 	printf(" -m: change merge_across_nodes tunable\n"
162 	       "     Default: %d\n", KSM_MERGE_ACROSS_NODES_DEFAULT);
163 	printf(" -d: turn debugging output on\n");
164 	printf(" -s: the size of duplicated memory area (in MiB)\n");
165 	printf(" -t: KSM merge type\n"
166 	       "     Default: 0\n"
167 	       "     0: madvise merging\n"
168 	       "     1: prctl merging\n");
169 
170 	exit(0);
171 }
172 
allocate_memory(void * ptr,int prot,int mapping,char data,size_t map_size)173 static void  *allocate_memory(void *ptr, int prot, int mapping, char data, size_t map_size)
174 {
175 	void *map_ptr = mmap(ptr, map_size, PROT_WRITE, mapping, -1, 0);
176 
177 	if (!map_ptr) {
178 		perror("mmap");
179 		return NULL;
180 	}
181 	memset(map_ptr, data, map_size);
182 	if (mprotect(map_ptr, map_size, prot)) {
183 		perror("mprotect");
184 		munmap(map_ptr, map_size);
185 		return NULL;
186 	}
187 
188 	return map_ptr;
189 }
190 
ksm_do_scan(int scan_count,struct timespec start_time,int timeout)191 static int ksm_do_scan(int scan_count, struct timespec start_time, int timeout)
192 {
193 	struct timespec cur_time;
194 	unsigned long cur_scan, init_scan;
195 
196 	if (ksm_read_sysfs(KSM_FP("full_scans"), &init_scan))
197 		return 1;
198 	cur_scan = init_scan;
199 
200 	while (cur_scan < init_scan + scan_count) {
201 		if (ksm_read_sysfs(KSM_FP("full_scans"), &cur_scan))
202 			return 1;
203 		if (clock_gettime(CLOCK_MONOTONIC_RAW, &cur_time)) {
204 			perror("clock_gettime");
205 			return 1;
206 		}
207 		if ((cur_time.tv_sec - start_time.tv_sec) > timeout) {
208 			printf("Scan time limit exceeded\n");
209 			return 1;
210 		}
211 	}
212 
213 	return 0;
214 }
215 
ksm_merge_pages(int merge_type,void * addr,size_t size,struct timespec start_time,int timeout)216 static int ksm_merge_pages(int merge_type, void *addr, size_t size,
217 			struct timespec start_time, int timeout)
218 {
219 	if (merge_type == KSM_MERGE_MADVISE) {
220 		if (madvise(addr, size, MADV_MERGEABLE)) {
221 			perror("madvise");
222 			return 1;
223 		}
224 	} else if (merge_type == KSM_MERGE_PRCTL) {
225 		if (prctl(PR_SET_MEMORY_MERGE, 1, 0, 0, 0)) {
226 			perror("prctl");
227 			return 1;
228 		}
229 	}
230 
231 	if (ksm_write_sysfs(KSM_FP("run"), 1))
232 		return 1;
233 
234 	/* Since merging occurs only after 2 scans, make sure to get at least 2 full scans */
235 	if (ksm_do_scan(2, start_time, timeout))
236 		return 1;
237 
238 	return 0;
239 }
240 
ksm_unmerge_pages(void * addr,size_t size,struct timespec start_time,int timeout)241 static int ksm_unmerge_pages(void *addr, size_t size,
242 			     struct timespec start_time, int timeout)
243 {
244 	if (madvise(addr, size, MADV_UNMERGEABLE)) {
245 		perror("madvise");
246 		return 1;
247 	}
248 	return 0;
249 }
250 
assert_ksm_pages_count(long dupl_page_count)251 static bool assert_ksm_pages_count(long dupl_page_count)
252 {
253 	unsigned long max_page_sharing, pages_sharing, pages_shared;
254 
255 	if (ksm_read_sysfs(KSM_FP("pages_shared"), &pages_shared) ||
256 	    ksm_read_sysfs(KSM_FP("pages_sharing"), &pages_sharing) ||
257 	    ksm_read_sysfs(KSM_FP("max_page_sharing"), &max_page_sharing))
258 		return false;
259 
260 	if (debug) {
261 		ksm_print_sysfs();
262 		ksm_print_procfs();
263 	}
264 
265 	/*
266 	 * Since there must be at least 2 pages for merging and 1 page can be
267 	 * shared with the limited number of pages (max_page_sharing), sometimes
268 	 * there are 'leftover' pages that cannot be merged. For example, if there
269 	 * are 11 pages and max_page_sharing = 10, then only 10 pages will be
270 	 * merged and the 11th page won't be affected. As a result, when the number
271 	 * of duplicate pages is divided by max_page_sharing and the remainder is 1,
272 	 * pages_shared and pages_sharing values will be equal between dupl_page_count
273 	 * and dupl_page_count - 1.
274 	 */
275 	if (dupl_page_count % max_page_sharing == 1 || dupl_page_count % max_page_sharing == 0) {
276 		if (pages_shared == dupl_page_count / max_page_sharing &&
277 		    pages_sharing == pages_shared * (max_page_sharing - 1))
278 			return true;
279 	} else {
280 		if (pages_shared == (dupl_page_count / max_page_sharing + 1) &&
281 		    pages_sharing == dupl_page_count - pages_shared)
282 			return true;
283 	}
284 
285 	return false;
286 }
287 
ksm_save_def(struct ksm_sysfs * ksm_sysfs)288 static int ksm_save_def(struct ksm_sysfs *ksm_sysfs)
289 {
290 	if (ksm_read_sysfs(KSM_FP("max_page_sharing"), &ksm_sysfs->max_page_sharing) ||
291 	    numa_available() ? 0 :
292 		ksm_read_sysfs(KSM_FP("merge_across_nodes"), &ksm_sysfs->merge_across_nodes) ||
293 	    ksm_read_sysfs(KSM_FP("sleep_millisecs"), &ksm_sysfs->sleep_millisecs) ||
294 	    ksm_read_sysfs(KSM_FP("pages_to_scan"), &ksm_sysfs->pages_to_scan) ||
295 	    ksm_read_sysfs(KSM_FP("run"), &ksm_sysfs->run) ||
296 	    ksm_read_sysfs(KSM_FP("stable_node_chains_prune_millisecs"),
297 			   &ksm_sysfs->stable_node_chains_prune_millisecs) ||
298 	    ksm_read_sysfs(KSM_FP("use_zero_pages"), &ksm_sysfs->use_zero_pages))
299 		return 1;
300 
301 	return 0;
302 }
303 
ksm_restore(struct ksm_sysfs * ksm_sysfs)304 static int ksm_restore(struct ksm_sysfs *ksm_sysfs)
305 {
306 	if (ksm_write_sysfs(KSM_FP("max_page_sharing"), ksm_sysfs->max_page_sharing) ||
307 	    numa_available() ? 0 :
308 		ksm_write_sysfs(KSM_FP("merge_across_nodes"), ksm_sysfs->merge_across_nodes) ||
309 	    ksm_write_sysfs(KSM_FP("pages_to_scan"), ksm_sysfs->pages_to_scan) ||
310 	    ksm_write_sysfs(KSM_FP("run"), ksm_sysfs->run) ||
311 	    ksm_write_sysfs(KSM_FP("sleep_millisecs"), ksm_sysfs->sleep_millisecs) ||
312 	    ksm_write_sysfs(KSM_FP("stable_node_chains_prune_millisecs"),
313 			    ksm_sysfs->stable_node_chains_prune_millisecs) ||
314 	    ksm_write_sysfs(KSM_FP("use_zero_pages"), ksm_sysfs->use_zero_pages))
315 		return 1;
316 
317 	return 0;
318 }
319 
check_ksm_merge(int merge_type,int mapping,int prot,long page_count,int timeout,size_t page_size)320 static int check_ksm_merge(int merge_type, int mapping, int prot,
321 			long page_count, int timeout, size_t page_size)
322 {
323 	void *map_ptr;
324 	struct timespec start_time;
325 
326 	if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
327 		perror("clock_gettime");
328 		return KSFT_FAIL;
329 	}
330 
331 	/* fill pages with the same data and merge them */
332 	map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count);
333 	if (!map_ptr)
334 		return KSFT_FAIL;
335 
336 	if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout))
337 		goto err_out;
338 
339 	/* verify that the right number of pages are merged */
340 	if (assert_ksm_pages_count(page_count)) {
341 		printf("OK\n");
342 		munmap(map_ptr, page_size * page_count);
343 		if (merge_type == KSM_MERGE_PRCTL)
344 			prctl(PR_SET_MEMORY_MERGE, 0, 0, 0, 0);
345 		return KSFT_PASS;
346 	}
347 
348 err_out:
349 	printf("Not OK\n");
350 	munmap(map_ptr, page_size * page_count);
351 	return KSFT_FAIL;
352 }
353 
check_ksm_unmerge(int merge_type,int mapping,int prot,int timeout,size_t page_size)354 static int check_ksm_unmerge(int merge_type, int mapping, int prot, int timeout, size_t page_size)
355 {
356 	void *map_ptr;
357 	struct timespec start_time;
358 	int page_count = 2;
359 
360 	if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
361 		perror("clock_gettime");
362 		return KSFT_FAIL;
363 	}
364 
365 	/* fill pages with the same data and merge them */
366 	map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count);
367 	if (!map_ptr)
368 		return KSFT_FAIL;
369 
370 	if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout))
371 		goto err_out;
372 
373 	/* change 1 byte in each of the 2 pages -- KSM must automatically unmerge them */
374 	memset(map_ptr, '-', 1);
375 	memset(map_ptr + page_size, '+', 1);
376 
377 	/* get at least 1 scan, so KSM can detect that the pages were modified */
378 	if (ksm_do_scan(1, start_time, timeout))
379 		goto err_out;
380 
381 	/* check that unmerging was successful and 0 pages are currently merged */
382 	if (assert_ksm_pages_count(0)) {
383 		printf("OK\n");
384 		munmap(map_ptr, page_size * page_count);
385 		return KSFT_PASS;
386 	}
387 
388 err_out:
389 	printf("Not OK\n");
390 	munmap(map_ptr, page_size * page_count);
391 	return KSFT_FAIL;
392 }
393 
check_ksm_zero_page_merge(int merge_type,int mapping,int prot,long page_count,int timeout,bool use_zero_pages,size_t page_size)394 static int check_ksm_zero_page_merge(int merge_type, int mapping, int prot, long page_count,
395 				int timeout, bool use_zero_pages, size_t page_size)
396 {
397 	void *map_ptr;
398 	struct timespec start_time;
399 
400 	if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
401 		perror("clock_gettime");
402 		return KSFT_FAIL;
403 	}
404 
405 	if (ksm_write_sysfs(KSM_FP("use_zero_pages"), use_zero_pages))
406 		return KSFT_FAIL;
407 
408 	/* fill pages with zero and try to merge them */
409 	map_ptr = allocate_memory(NULL, prot, mapping, 0, page_size * page_count);
410 	if (!map_ptr)
411 		return KSFT_FAIL;
412 
413 	if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout))
414 		goto err_out;
415 
416        /*
417 	* verify that the right number of pages are merged:
418 	* 1) if use_zero_pages is set to 1, empty pages are merged
419 	*    with the kernel zero page instead of with each other;
420 	* 2) if use_zero_pages is set to 0, empty pages are not treated specially
421 	*    and merged as usual.
422 	*/
423 	if (use_zero_pages && !assert_ksm_pages_count(0))
424 		goto err_out;
425 	else if (!use_zero_pages && !assert_ksm_pages_count(page_count))
426 		goto err_out;
427 
428 	printf("OK\n");
429 	munmap(map_ptr, page_size * page_count);
430 	return KSFT_PASS;
431 
432 err_out:
433 	printf("Not OK\n");
434 	munmap(map_ptr, page_size * page_count);
435 	return KSFT_FAIL;
436 }
437 
get_next_mem_node(int node)438 static int get_next_mem_node(int node)
439 {
440 
441 	long node_size;
442 	int mem_node = 0;
443 	int i, max_node = numa_max_node();
444 
445 	for (i = node + 1; i <= max_node + node; i++) {
446 		mem_node = i % (max_node + 1);
447 		node_size = numa_node_size(mem_node, NULL);
448 		if (node_size > 0)
449 			break;
450 	}
451 	return mem_node;
452 }
453 
get_first_mem_node(void)454 static int get_first_mem_node(void)
455 {
456 	return get_next_mem_node(numa_max_node());
457 }
458 
check_ksm_numa_merge(int merge_type,int mapping,int prot,int timeout,bool merge_across_nodes,size_t page_size)459 static int check_ksm_numa_merge(int merge_type, int mapping, int prot, int timeout,
460 				bool merge_across_nodes, size_t page_size)
461 {
462 	void *numa1_map_ptr, *numa2_map_ptr;
463 	struct timespec start_time;
464 	int page_count = 2;
465 	int first_node;
466 
467 	if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
468 		perror("clock_gettime");
469 		return KSFT_FAIL;
470 	}
471 
472 	if (numa_available() < 0) {
473 		perror("NUMA support not enabled");
474 		return KSFT_SKIP;
475 	}
476 	if (numa_num_configured_nodes() <= 1) {
477 		printf("At least 2 NUMA nodes must be available\n");
478 		return KSFT_SKIP;
479 	}
480 	if (ksm_write_sysfs(KSM_FP("merge_across_nodes"), merge_across_nodes))
481 		return KSFT_FAIL;
482 
483 	/* allocate 2 pages in 2 different NUMA nodes and fill them with the same data */
484 	first_node = get_first_mem_node();
485 	numa1_map_ptr = numa_alloc_onnode(page_size, first_node);
486 	numa2_map_ptr = numa_alloc_onnode(page_size, get_next_mem_node(first_node));
487 	if (!numa1_map_ptr || !numa2_map_ptr) {
488 		perror("numa_alloc_onnode");
489 		return KSFT_FAIL;
490 	}
491 
492 	memset(numa1_map_ptr, '*', page_size);
493 	memset(numa2_map_ptr, '*', page_size);
494 
495 	/* try to merge the pages */
496 	if (ksm_merge_pages(merge_type, numa1_map_ptr, page_size, start_time, timeout) ||
497 	    ksm_merge_pages(merge_type, numa2_map_ptr, page_size, start_time, timeout))
498 		goto err_out;
499 
500        /*
501 	* verify that the right number of pages are merged:
502 	* 1) if merge_across_nodes was enabled, 2 duplicate pages will be merged;
503 	* 2) if merge_across_nodes = 0, there must be 0 merged pages, since there is
504 	*    only 1 unique page in each node and they can't be shared.
505 	*/
506 	if (merge_across_nodes && !assert_ksm_pages_count(page_count))
507 		goto err_out;
508 	else if (!merge_across_nodes && !assert_ksm_pages_count(0))
509 		goto err_out;
510 
511 	numa_free(numa1_map_ptr, page_size);
512 	numa_free(numa2_map_ptr, page_size);
513 	printf("OK\n");
514 	return KSFT_PASS;
515 
516 err_out:
517 	numa_free(numa1_map_ptr, page_size);
518 	numa_free(numa2_map_ptr, page_size);
519 	printf("Not OK\n");
520 	return KSFT_FAIL;
521 }
522 
ksm_merge_hugepages_time(int merge_type,int mapping,int prot,int timeout,size_t map_size)523 static int ksm_merge_hugepages_time(int merge_type, int mapping, int prot,
524 				int timeout, size_t map_size)
525 {
526 	void *map_ptr, *map_ptr_orig;
527 	struct timespec start_time, end_time;
528 	unsigned long scan_time_ns;
529 	int pagemap_fd, n_normal_pages, n_huge_pages;
530 
531 	if (!thp_is_enabled()) {
532 		printf("Transparent Hugepages not available\n");
533 		return KSFT_SKIP;
534 	}
535 
536 	map_size *= MB;
537 	size_t len = map_size;
538 
539 	len -= len % HPAGE_SIZE;
540 	map_ptr_orig = mmap(NULL, len + HPAGE_SIZE, PROT_READ | PROT_WRITE,
541 			MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE, -1, 0);
542 	map_ptr = map_ptr_orig + HPAGE_SIZE - (uintptr_t)map_ptr_orig % HPAGE_SIZE;
543 
544 	if (map_ptr_orig == MAP_FAILED)
545 		err(2, "initial mmap");
546 
547 	if (madvise(map_ptr, len, MADV_HUGEPAGE))
548 		err(2, "MADV_HUGEPAGE");
549 
550 	pagemap_fd = open("/proc/self/pagemap", O_RDONLY);
551 	if (pagemap_fd < 0)
552 		err(2, "open pagemap");
553 
554 	n_normal_pages = 0;
555 	n_huge_pages = 0;
556 	for (void *p = map_ptr; p < map_ptr + len; p += HPAGE_SIZE) {
557 		if (allocate_transhuge(p, pagemap_fd) < 0)
558 			n_normal_pages++;
559 		else
560 			n_huge_pages++;
561 	}
562 	printf("Number of normal pages:    %d\n", n_normal_pages);
563 	printf("Number of huge pages:    %d\n", n_huge_pages);
564 
565 	memset(map_ptr, '*', len);
566 
567 	if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
568 		perror("clock_gettime");
569 		goto err_out;
570 	}
571 	if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout))
572 		goto err_out;
573 	if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) {
574 		perror("clock_gettime");
575 		goto err_out;
576 	}
577 
578 	scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC +
579 		       (end_time.tv_nsec - start_time.tv_nsec);
580 
581 	printf("Total size:    %lu MiB\n", map_size / MB);
582 	printf("Total time:    %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC,
583 	       scan_time_ns % NSEC_PER_SEC);
584 	printf("Average speed:  %.3f MiB/s\n", (map_size / MB) /
585 					       ((double)scan_time_ns / NSEC_PER_SEC));
586 
587 	munmap(map_ptr_orig, len + HPAGE_SIZE);
588 	return KSFT_PASS;
589 
590 err_out:
591 	printf("Not OK\n");
592 	munmap(map_ptr_orig, len + HPAGE_SIZE);
593 	return KSFT_FAIL;
594 }
595 
ksm_merge_time(int merge_type,int mapping,int prot,int timeout,size_t map_size)596 static int ksm_merge_time(int merge_type, int mapping, int prot, int timeout, size_t map_size)
597 {
598 	void *map_ptr;
599 	struct timespec start_time, end_time;
600 	unsigned long scan_time_ns;
601 
602 	map_size *= MB;
603 
604 	map_ptr = allocate_memory(NULL, prot, mapping, '*', map_size);
605 	if (!map_ptr)
606 		return KSFT_FAIL;
607 
608 	if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
609 		perror("clock_gettime");
610 		goto err_out;
611 	}
612 	if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout))
613 		goto err_out;
614 	if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) {
615 		perror("clock_gettime");
616 		goto err_out;
617 	}
618 
619 	scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC +
620 		       (end_time.tv_nsec - start_time.tv_nsec);
621 
622 	printf("Total size:    %lu MiB\n", map_size / MB);
623 	printf("Total time:    %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC,
624 	       scan_time_ns % NSEC_PER_SEC);
625 	printf("Average speed:  %.3f MiB/s\n", (map_size / MB) /
626 					       ((double)scan_time_ns / NSEC_PER_SEC));
627 
628 	munmap(map_ptr, map_size);
629 	return KSFT_PASS;
630 
631 err_out:
632 	printf("Not OK\n");
633 	munmap(map_ptr, map_size);
634 	return KSFT_FAIL;
635 }
636 
ksm_unmerge_time(int merge_type,int mapping,int prot,int timeout,size_t map_size)637 static int ksm_unmerge_time(int merge_type, int mapping, int prot, int timeout, size_t map_size)
638 {
639 	void *map_ptr;
640 	struct timespec start_time, end_time;
641 	unsigned long scan_time_ns;
642 
643 	map_size *= MB;
644 
645 	map_ptr = allocate_memory(NULL, prot, mapping, '*', map_size);
646 	if (!map_ptr)
647 		return KSFT_FAIL;
648 	if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
649 		perror("clock_gettime");
650 		goto err_out;
651 	}
652 	if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout))
653 		goto err_out;
654 
655 	if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
656 		perror("clock_gettime");
657 		goto err_out;
658 	}
659 	if (ksm_unmerge_pages(map_ptr, map_size, start_time, timeout))
660 		goto err_out;
661 	if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) {
662 		perror("clock_gettime");
663 		goto err_out;
664 	}
665 
666 	scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC +
667 		       (end_time.tv_nsec - start_time.tv_nsec);
668 
669 	printf("Total size:    %lu MiB\n", map_size / MB);
670 	printf("Total time:    %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC,
671 	       scan_time_ns % NSEC_PER_SEC);
672 	printf("Average speed:  %.3f MiB/s\n", (map_size / MB) /
673 					       ((double)scan_time_ns / NSEC_PER_SEC));
674 
675 	munmap(map_ptr, map_size);
676 	return KSFT_PASS;
677 
678 err_out:
679 	printf("Not OK\n");
680 	munmap(map_ptr, map_size);
681 	return KSFT_FAIL;
682 }
683 
ksm_cow_time(int merge_type,int mapping,int prot,int timeout,size_t page_size)684 static int ksm_cow_time(int merge_type, int mapping, int prot, int timeout, size_t page_size)
685 {
686 	void *map_ptr;
687 	struct timespec start_time, end_time;
688 	unsigned long cow_time_ns;
689 
690 	/* page_count must be less than 2*page_size */
691 	size_t page_count = 4000;
692 
693 	map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count);
694 	if (!map_ptr)
695 		return KSFT_FAIL;
696 
697 	if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
698 		perror("clock_gettime");
699 		return KSFT_FAIL;
700 	}
701 	for (size_t i = 0; i < page_count - 1; i = i + 2)
702 		memset(map_ptr + page_size * i, '-', 1);
703 	if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) {
704 		perror("clock_gettime");
705 		return KSFT_FAIL;
706 	}
707 
708 	cow_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC +
709 		       (end_time.tv_nsec - start_time.tv_nsec);
710 
711 	printf("Total size:    %lu MiB\n\n", (page_size * page_count) / MB);
712 	printf("Not merged pages:\n");
713 	printf("Total time:     %ld.%09ld s\n", cow_time_ns / NSEC_PER_SEC,
714 	       cow_time_ns % NSEC_PER_SEC);
715 	printf("Average speed:  %.3f MiB/s\n\n", ((page_size * (page_count / 2)) / MB) /
716 					       ((double)cow_time_ns / NSEC_PER_SEC));
717 
718 	/* Create 2000 pairs of duplicate pages */
719 	for (size_t i = 0; i < page_count - 1; i = i + 2) {
720 		memset(map_ptr + page_size * i, '+', i / 2 + 1);
721 		memset(map_ptr + page_size * (i + 1), '+', i / 2 + 1);
722 	}
723 	if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout))
724 		goto err_out;
725 
726 	if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
727 		perror("clock_gettime");
728 		goto err_out;
729 	}
730 	for (size_t i = 0; i < page_count - 1; i = i + 2)
731 		memset(map_ptr + page_size * i, '-', 1);
732 	if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) {
733 		perror("clock_gettime");
734 		goto err_out;
735 	}
736 
737 	cow_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC +
738 		       (end_time.tv_nsec - start_time.tv_nsec);
739 
740 	printf("Merged pages:\n");
741 	printf("Total time:     %ld.%09ld s\n", cow_time_ns / NSEC_PER_SEC,
742 	       cow_time_ns % NSEC_PER_SEC);
743 	printf("Average speed:  %.3f MiB/s\n", ((page_size * (page_count / 2)) / MB) /
744 					       ((double)cow_time_ns / NSEC_PER_SEC));
745 
746 	munmap(map_ptr, page_size * page_count);
747 	return KSFT_PASS;
748 
749 err_out:
750 	printf("Not OK\n");
751 	munmap(map_ptr, page_size * page_count);
752 	return KSFT_FAIL;
753 }
754 
main(int argc,char * argv[])755 int main(int argc, char *argv[])
756 {
757 	int ret = 0, opt;
758 	int prot = 0;
759 	int ksm_scan_limit_sec = KSM_SCAN_LIMIT_SEC_DEFAULT;
760 	int merge_type = KSM_MERGE_TYPE_DEFAULT;
761 	long page_count = KSM_PAGE_COUNT_DEFAULT;
762 	size_t page_size = sysconf(_SC_PAGESIZE);
763 	struct ksm_sysfs ksm_sysfs_old;
764 	int test_name = CHECK_KSM_MERGE;
765 	bool use_zero_pages = KSM_USE_ZERO_PAGES_DEFAULT;
766 	bool merge_across_nodes = KSM_MERGE_ACROSS_NODES_DEFAULT;
767 	long size_MB = 0;
768 
769 	while ((opt = getopt(argc, argv, "dha:p:l:z:m:s:t:MUZNPCHD")) != -1) {
770 		switch (opt) {
771 		case 'a':
772 			prot = str_to_prot(optarg);
773 			break;
774 		case 'p':
775 			page_count = atol(optarg);
776 			if (page_count <= 0) {
777 				printf("The number of pages must be greater than 0\n");
778 				return KSFT_FAIL;
779 			}
780 			break;
781 		case 'l':
782 			ksm_scan_limit_sec = atoi(optarg);
783 			if (ksm_scan_limit_sec <= 0) {
784 				printf("Timeout value must be greater than 0\n");
785 				return KSFT_FAIL;
786 			}
787 			break;
788 		case 'h':
789 			print_help();
790 			break;
791 		case 'z':
792 			if (strcmp(optarg, "0") == 0)
793 				use_zero_pages = 0;
794 			else
795 				use_zero_pages = 1;
796 			break;
797 		case 'm':
798 			if (strcmp(optarg, "0") == 0)
799 				merge_across_nodes = 0;
800 			else
801 				merge_across_nodes = 1;
802 			break;
803 		case 'd':
804 			debug = 1;
805 			break;
806 		case 's':
807 			size_MB = atoi(optarg);
808 			if (size_MB <= 0) {
809 				printf("Size must be greater than 0\n");
810 				return KSFT_FAIL;
811 			}
812 			break;
813 		case 't':
814 			{
815 				int tmp = atoi(optarg);
816 
817 				if (tmp < 0 || tmp > KSM_MERGE_LAST) {
818 					printf("Invalid merge type\n");
819 					return KSFT_FAIL;
820 				}
821 				merge_type = tmp;
822 			}
823 			break;
824 		case 'M':
825 			break;
826 		case 'U':
827 			test_name = CHECK_KSM_UNMERGE;
828 			break;
829 		case 'Z':
830 			test_name = CHECK_KSM_ZERO_PAGE_MERGE;
831 			break;
832 		case 'N':
833 			test_name = CHECK_KSM_NUMA_MERGE;
834 			break;
835 		case 'P':
836 			test_name = KSM_MERGE_TIME;
837 			break;
838 		case 'H':
839 			test_name = KSM_MERGE_TIME_HUGE_PAGES;
840 			break;
841 		case 'D':
842 			test_name = KSM_UNMERGE_TIME;
843 			break;
844 		case 'C':
845 			test_name = KSM_COW_TIME;
846 			break;
847 		default:
848 			return KSFT_FAIL;
849 		}
850 	}
851 
852 	if (prot == 0)
853 		prot = str_to_prot(KSM_PROT_STR_DEFAULT);
854 
855 	if (access(KSM_SYSFS_PATH, F_OK)) {
856 		printf("Config KSM not enabled\n");
857 		return KSFT_SKIP;
858 	}
859 
860 	if (ksm_save_def(&ksm_sysfs_old)) {
861 		printf("Cannot save default tunables\n");
862 		return KSFT_FAIL;
863 	}
864 
865 	if (ksm_write_sysfs(KSM_FP("run"), 2) ||
866 	    ksm_write_sysfs(KSM_FP("sleep_millisecs"), 0) ||
867 	    numa_available() ? 0 :
868 		ksm_write_sysfs(KSM_FP("merge_across_nodes"), 1) ||
869 	    ksm_write_sysfs(KSM_FP("pages_to_scan"), page_count))
870 		return KSFT_FAIL;
871 
872 	switch (test_name) {
873 	case CHECK_KSM_MERGE:
874 		ret = check_ksm_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, page_count,
875 				      ksm_scan_limit_sec, page_size);
876 		break;
877 	case CHECK_KSM_UNMERGE:
878 		ret = check_ksm_unmerge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,
879 					ksm_scan_limit_sec, page_size);
880 		break;
881 	case CHECK_KSM_ZERO_PAGE_MERGE:
882 		ret = check_ksm_zero_page_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,
883 						page_count, ksm_scan_limit_sec, use_zero_pages,
884 						page_size);
885 		break;
886 	case CHECK_KSM_NUMA_MERGE:
887 		ret = check_ksm_numa_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,
888 					ksm_scan_limit_sec, merge_across_nodes, page_size);
889 		break;
890 	case KSM_MERGE_TIME:
891 		if (size_MB == 0) {
892 			printf("Option '-s' is required.\n");
893 			return KSFT_FAIL;
894 		}
895 		ret = ksm_merge_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,
896 				ksm_scan_limit_sec, size_MB);
897 		break;
898 	case KSM_MERGE_TIME_HUGE_PAGES:
899 		if (size_MB == 0) {
900 			printf("Option '-s' is required.\n");
901 			return KSFT_FAIL;
902 		}
903 		ret = ksm_merge_hugepages_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,
904 				ksm_scan_limit_sec, size_MB);
905 		break;
906 	case KSM_UNMERGE_TIME:
907 		if (size_MB == 0) {
908 			printf("Option '-s' is required.\n");
909 			return KSFT_FAIL;
910 		}
911 		ret = ksm_unmerge_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,
912 				       ksm_scan_limit_sec, size_MB);
913 		break;
914 	case KSM_COW_TIME:
915 		ret = ksm_cow_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,
916 				ksm_scan_limit_sec, page_size);
917 		break;
918 	}
919 
920 	if (ksm_restore(&ksm_sysfs_old)) {
921 		printf("Cannot restore default tunables\n");
922 		return KSFT_FAIL;
923 	}
924 
925 	return ret;
926 }
927