xref: /linux/tools/testing/selftests/mm/khugepaged.c (revision 49c77c5de353f47c9607ba9ef9a6748a361ad975)
1 #define _GNU_SOURCE
2 #include <ctype.h>
3 #include <errno.h>
4 #include <fcntl.h>
5 #include <limits.h>
6 #include <dirent.h>
7 #include <signal.h>
8 #include <stdio.h>
9 #include <stdlib.h>
10 #include <stdbool.h>
11 #include <string.h>
12 #include <unistd.h>
13 
14 #include <linux/mman.h>
15 #include <sys/mman.h>
16 #include <sys/wait.h>
17 #include <sys/types.h>
18 #include <sys/stat.h>
19 #include <sys/sysmacros.h>
20 #include <sys/vfs.h>
21 
22 #include "linux/magic.h"
23 
24 #include "vm_util.h"
25 
26 #define BASE_ADDR ((void *)(1UL << 30))
27 static unsigned long hpage_pmd_size;
28 static int hpage_pmd_order;
29 static unsigned long page_size;
30 static int hpage_pmd_nr;
31 static int anon_order;
32 static int collapse_order;
33 static bool collapse_order_set;
34 static int collapse_orders[NR_ORDERS];
35 static int nr_collapse_orders;
36 static int pagemap_fd = -1;
37 static int kpageflags_fd = -1;
38 
39 #define PID_SMAPS "/proc/self/smaps"
40 #define TEST_FILE "collapse_test_file"
41 
42 #define MAX_LINE_LENGTH 500
43 
44 enum vma_type {
45 	VMA_ANON,
46 	VMA_FILE,
47 	VMA_SHMEM,
48 };
49 
50 enum file_setup_ops {
51 	FILE_SETUP_READ_ONLY_FS,
52 	FILE_SETUP_READ_WRITE_FS_READ_DATA,
53 	FILE_SETUP_READ_WRITE_FS_WRITE_DATA,
54 };
55 
56 struct mem_ops {
57 	void *(*setup_area)(int nr_hpages);
58 	void (*cleanup_area)(void *p, unsigned long size);
59 	void (*fault)(void *p, unsigned long start, unsigned long end);
60 	bool (*check_huge)(void *addr, size_t len, int nr_hpages, uint64_t hpage_size);
61 	const char *name;
62 };
63 
64 static struct mem_ops *read_only_file_ops;
65 static struct mem_ops *read_write_file_read_ops;
66 static struct mem_ops *read_write_file_write_ops;
67 static struct mem_ops *anon_ops;
68 static struct mem_ops *shmem_ops;
69 
70 struct collapse_context {
71 	void (*collapse)(const char *msg, char *p, int nr_hpages,
72 			 struct mem_ops *ops, bool expect);
73 	bool enforce_pte_scan_limits;
74 	const char *name;
75 };
76 
77 static struct collapse_context *khugepaged_context;
78 static struct collapse_context *mthp_khugepaged_context;
79 static struct collapse_context *madvise_context;
80 
81 struct file_info {
82 	const char *dir;
83 	char path[PATH_MAX];
84 	enum vma_type type;
85 	int fd;
86 	char dev_queue_read_ahead_path[PATH_MAX];
87 };
88 
89 static struct file_info finfo;
90 static int exit_status;
91 
92 static void success(const char *msg)
93 {
94 	printf(" \e[32m%s\e[0m\n", msg);
95 	exit_status = KSFT_PASS;
96 }
97 
98 static void fail(const char *msg)
99 {
100 	printf(" \e[31m%s\e[0m\n", msg);
101 	exit_status = KSFT_FAIL;
102 }
103 
104 static void skip(const char *msg)
105 {
106 	printf(" \e[33m%s\e[0m\n", msg);
107 	exit_status = KSFT_SKIP;
108 }
109 
110 static void save_settings(void)
111 {
112 	ksft_print_msg("Save THP and khugepaged settings...");
113 	if ((read_only_file_ops || read_write_file_read_ops ||
114 	     read_write_file_write_ops) &&
115 	    finfo.type == VMA_FILE)
116 		thp_set_read_ahead_path(finfo.dev_queue_read_ahead_path);
117 	thp_save_settings();
118 
119 	success("OK");
120 }
121 
122 static void get_finfo(const char *dir)
123 {
124 	struct stat path_stat;
125 	struct statfs fs;
126 	char buf[1 << 10];
127 	char path[PATH_MAX];
128 	char *str, *end;
129 	int ret;
130 
131 	finfo.dir = dir;
132 	if (stat(finfo.dir, &path_stat))
133 		ksft_exit_fail_perror("stat()");
134 	if (!S_ISDIR(path_stat.st_mode))
135 		ksft_exit_fail_msg("%s: Not a directory (%s)\n", __func__, finfo.dir);
136 	if (snprintf(finfo.path, sizeof(finfo.path), "%s/" TEST_FILE,
137 		     finfo.dir) >= sizeof(finfo.path))
138 		ksft_exit_fail_msg("%s: Pathname is too long\n", __func__);
139 	if (statfs(finfo.dir, &fs))
140 		ksft_exit_fail_perror("statfs()");
141 	finfo.type = fs.f_type == TMPFS_MAGIC ? VMA_SHMEM : VMA_FILE;
142 	if (finfo.type == VMA_SHMEM)
143 		return;
144 
145 	/* Find owning device's queue/read_ahead_kb control */
146 	if (snprintf(path, sizeof(path), "/sys/dev/block/%d:%d/uevent",
147 		     major(path_stat.st_dev), minor(path_stat.st_dev))
148 	    >= sizeof(path))
149 		ksft_exit_fail_msg("%s: Pathname is too long\n", __func__);
150 	ret = read_file(path, buf, sizeof(buf));
151 	if (ret)
152 		ksft_exit_fail_msg("read_file(%s): %s\n", path, strerror(-ret));
153 	if (strstr(buf, "DEVTYPE=disk")) {
154 		/* Found it */
155 		if (snprintf(finfo.dev_queue_read_ahead_path,
156 			     sizeof(finfo.dev_queue_read_ahead_path),
157 			     "/sys/dev/block/%d:%d/queue/read_ahead_kb",
158 			     major(path_stat.st_dev), minor(path_stat.st_dev))
159 		    >= sizeof(finfo.dev_queue_read_ahead_path))
160 			ksft_exit_fail_msg("%s: Pathname is too long\n", __func__);
161 		return;
162 	}
163 	if (!strstr(buf, "DEVTYPE=partition"))
164 		ksft_exit_fail_msg("%s: Unknown device type: %s\n", __func__, path);
165 	/*
166 	 * Partition of block device - need to find actual device.
167 	 * Using naming convention that devnameN is partition of
168 	 * device devname.
169 	 */
170 	str = strstr(buf, "DEVNAME=");
171 	if (!str)
172 		ksft_exit_fail_msg("%s: Could not read: %s", __func__, path);
173 	str += 8;
174 	end = str;
175 	while (*end) {
176 		if (isdigit(*end)) {
177 			*end = '\0';
178 			if (snprintf(finfo.dev_queue_read_ahead_path,
179 				     sizeof(finfo.dev_queue_read_ahead_path),
180 				     "/sys/block/%s/queue/read_ahead_kb",
181 				     str) >= sizeof(finfo.dev_queue_read_ahead_path))
182 				ksft_exit_fail_msg("%s: Pathname is too long\n", __func__);
183 			return;
184 		}
185 		++end;
186 	}
187 	ksft_exit_fail_msg("%s: Could not read: %s\n", __func__, path);
188 }
189 
190 static bool check_swap(void *addr, unsigned long size)
191 {
192 	bool swap = false;
193 	int ret;
194 	FILE *fp;
195 	char buffer[MAX_LINE_LENGTH];
196 	char addr_pattern[MAX_LINE_LENGTH];
197 
198 	ret = snprintf(addr_pattern, MAX_LINE_LENGTH, "%08lx-",
199 		       (unsigned long) addr);
200 	if (ret >= MAX_LINE_LENGTH)
201 		ksft_exit_fail_msg("%s: Pattern is too long\n", __func__);
202 
203 	fp = fopen(PID_SMAPS, "r");
204 	if (!fp)
205 		ksft_exit_fail_msg("%s: Failed to open file %s\n", __func__, PID_SMAPS);
206 	if (!check_for_pattern(fp, addr_pattern, buffer, sizeof(buffer)))
207 		goto err_out;
208 
209 	ret = snprintf(addr_pattern, MAX_LINE_LENGTH, "Swap:%19ld kB",
210 		       size >> 10);
211 	if (ret >= MAX_LINE_LENGTH)
212 		ksft_exit_fail_msg("%s: Pattern is too long\n", __func__);
213 	/*
214 	 * Fetch the Swap: in the same block and check whether it got
215 	 * the expected number of hugeepages next.
216 	 */
217 	if (!check_for_pattern(fp, "Swap:", buffer, sizeof(buffer)))
218 		goto err_out;
219 
220 	if (strncmp(buffer, addr_pattern, strlen(addr_pattern)))
221 		goto err_out;
222 
223 	swap = true;
224 err_out:
225 	fclose(fp);
226 	return swap;
227 }
228 
229 static bool swapout_range(void *p, unsigned long size)
230 {
231 	int i;
232 
233 	/* keep khugepaged from collapsing the range and swapping it back in */
234 	if (madvise(p, size, MADV_NOHUGEPAGE))
235 		ksft_exit_fail_perror("madvise(MADV_NOHUGEPAGE)");
236 
237 	/*
238 	 * Retry several times because MADV_PAGEOUT is best effort.  Sleep
239 	 * between the retries to give outstanding writeback a chance to
240 	 * finish.
241 	 */
242 	for (i = 0; i < 40; i++) {
243 		if (madvise(p, size, MADV_PAGEOUT))
244 			ksft_exit_fail_perror("madvise(MADV_PAGEOUT)");
245 		if (check_swap(p, size))
246 			return true;
247 		usleep(50 * 1000);
248 	}
249 	return false;
250 }
251 
252 static void *alloc_mapping(int nr)
253 {
254 	void *p;
255 
256 	p = mmap(BASE_ADDR, nr * hpage_pmd_size, PROT_READ | PROT_WRITE,
257 		 MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
258 	if (p != BASE_ADDR)
259 		ksft_exit_fail_msg("Failed to allocate VMA at %p\n", BASE_ADDR);
260 
261 	return p;
262 }
263 
264 static void fill_memory(int *p, unsigned long start, unsigned long end)
265 {
266 	int i;
267 
268 	for (i = start / page_size; i < end / page_size; i++)
269 		p[i * page_size / sizeof(*p)] = i + 0xdead0000;
270 }
271 
272 /*
273  * MADV_COLLAPSE is a best-effort request and may fail if an internal
274  * resource is temporarily unavailable, in which case it will set errno to
275  * EAGAIN.  In such a case, immediately reattempt the operation one more
276  * time.
277  */
278 static int madvise_collapse_retry(void *p, unsigned long size)
279 {
280 	bool retry = true;
281 	int ret;
282 
283 retry:
284 	ret = madvise(p, size, MADV_COLLAPSE);
285 	if (ret && errno == EAGAIN && retry) {
286 		retry = false;
287 		goto retry;
288 	}
289 	return ret;
290 }
291 
292 /*
293  * Returns pmd-mapped hugepage in VMA marked VM_HUGEPAGE, filled with
294  * validate_memory()'able contents.
295  */
296 static void *alloc_hpage(struct mem_ops *ops)
297 {
298 	void *p = ops->setup_area(1);
299 
300 	ops->fault(p, 0, hpage_pmd_size);
301 
302 	/*
303 	 * VMA should be neither VM_HUGEPAGE nor VM_NOHUGEPAGE.
304 	 * The latter is ineligible for collapse by MADV_COLLAPSE
305 	 * while the former might cause MADV_COLLAPSE to race with
306 	 * khugepaged on low-load system (like a test machine), which
307 	 * would cause MADV_COLLAPSE to fail with EAGAIN.
308 	 */
309 	ksft_print_msg("Allocate huge page...");
310 	if (madvise_collapse_retry(p, hpage_pmd_size))
311 		ksft_exit_fail_perror("madvise(MADV_COLLAPSE)");
312 	if (!ops->check_huge(p, hpage_pmd_size, 1, hpage_pmd_size))
313 		ksft_exit_fail_perror("madvise(MADV_COLLAPSE)");
314 	if (madvise(p, hpage_pmd_size, MADV_HUGEPAGE))
315 		ksft_exit_fail_perror("madvise(MADV_HUGEPAGE)");
316 	success("OK");
317 	return p;
318 }
319 
320 static void validate_memory(int *p, unsigned long start, unsigned long end)
321 {
322 	int i;
323 
324 	for (i = start / page_size; i < end / page_size; i++) {
325 		if (p[i * page_size / sizeof(*p)] != i + 0xdead0000)
326 			ksft_exit_fail_msg("Page %d is corrupted: %#x\n",
327 					   i, p[i * page_size / sizeof(*p)]);
328 	}
329 }
330 
331 static void *anon_setup_area(int nr_hpages)
332 {
333 	return alloc_mapping(nr_hpages);
334 }
335 
336 static void anon_cleanup_area(void *p, unsigned long size)
337 {
338 	munmap(p, size);
339 }
340 
341 static void anon_fault(void *p, unsigned long start, unsigned long end)
342 {
343 	fill_memory(p, start, end);
344 }
345 
346 static void *file_setup_area_common(int nr_hpages, enum file_setup_ops setup)
347 {
348 	const int open_opt = setup == FILE_SETUP_READ_ONLY_FS ? O_RDONLY : O_RDWR;
349 	const int mmap_prot = setup == FILE_SETUP_READ_ONLY_FS ? PROT_READ : (PROT_READ | PROT_WRITE);
350 	int fd, ret;
351 	void *p;
352 	unsigned long size;
353 
354 	unlink(finfo.path);  /* Cleanup from previous failed tests */
355 	ksft_print_msg("Creating %s for collapse%s...", finfo.path,
356 		       finfo.type == VMA_SHMEM ? " (tmpfs)" : "");
357 	fd = open(finfo.path, O_CREAT | O_RDWR | O_TRUNC | O_EXCL,
358 		  777);
359 	if (fd < 0)
360 		ksft_exit_fail_perror("open()");
361 
362 	size = nr_hpages * hpage_pmd_size;
363 	if (ftruncate(fd, size))
364 		ksft_exit_fail_perror("ftruncate()");
365 	p = mmap(BASE_ADDR, size, PROT_READ | PROT_WRITE,
366 		MAP_SHARED, fd, 0);
367 	if (p != BASE_ADDR)
368 		ksft_exit_fail_perror("mmap()");
369 	fill_memory(p, 0, size);
370 	if (msync(p, size, MS_SYNC))
371 		ksft_exit_fail_perror("msync()");
372 	close(fd);
373 	munmap(p, size);
374 	success("OK");
375 	ksft_print_msg("Opening %s %s for collapse...", finfo.path,
376 	       setup == FILE_SETUP_READ_ONLY_FS ? "read-only" :
377 	       setup == FILE_SETUP_READ_WRITE_FS_READ_DATA ?
378 						  "read-write (read)" :
379 						  "read-write (write)");
380 	finfo.fd = open(finfo.path, open_opt, 777);
381 	if (finfo.fd < 0)
382 		ksft_exit_fail_perror("open()");
383 	p = mmap(BASE_ADDR, size, mmap_prot, MAP_SHARED, finfo.fd, 0);
384 	if (p == MAP_FAILED || p != BASE_ADDR)
385 		ksft_exit_fail_perror("mmap()");
386 
387 	/* Drop page cache */
388 	ret = write_file("/proc/sys/vm/drop_caches", "3", 2);
389 	if (ret)
390 		ksft_exit_fail_msg("write_file(drop_caches): %s\n",
391 				   strerror(-ret));
392 
393 	success("OK");
394 	return p;
395 }
396 
397 static void *file_setup_read_only_area(int nr_hpages)
398 {
399 	return file_setup_area_common(nr_hpages, FILE_SETUP_READ_ONLY_FS);
400 }
401 
402 static void *file_setup_read_write_fs_read_area(int nr_hpages)
403 {
404 	return file_setup_area_common(nr_hpages, FILE_SETUP_READ_WRITE_FS_READ_DATA);
405 }
406 
407 static void *file_setup_read_write_fs_write_area(int nr_hpages)
408 {
409 	return file_setup_area_common(nr_hpages, FILE_SETUP_READ_WRITE_FS_WRITE_DATA);
410 }
411 
412 static void file_cleanup_area(void *p, unsigned long size)
413 {
414 	munmap(p, size);
415 	close(finfo.fd);
416 	unlink(finfo.path);
417 }
418 
419 static void file_fault_read(void *p, unsigned long start, unsigned long end)
420 {
421 	if (madvise(((char *)p) + start, end - start, MADV_POPULATE_READ))
422 		ksft_exit_fail_perror("madvise(MADV_POPULATE_READ)");
423 }
424 
425 static void file_fault_read_and_flush(void *p, unsigned long start, unsigned long end)
426 {
427 	file_fault_read(p, start, end);
428 	/*
429 	 * make folio clean, since dirty folios from read&write file are
430 	 * rejected and not flushed
431 	 */
432 	msync((char *)p + start, end - start, MS_SYNC);
433 }
434 
435 static void file_fault_write(void *p, unsigned long start, unsigned long end)
436 {
437 	if (madvise(((char *)p) + start, end - start, MADV_POPULATE_WRITE))
438 		ksft_exit_fail_perror("madvise(MADV_POPULATE_WRITE)");
439 }
440 
441 static void *shmem_setup_area(int nr_hpages)
442 {
443 	void *p;
444 	unsigned long size = nr_hpages * hpage_pmd_size;
445 
446 	finfo.fd = memfd_create("khugepaged-selftest-collapse-shmem", 0);
447 	if (finfo.fd < 0)
448 		ksft_exit_fail_perror("memfd_create()");
449 	if (ftruncate(finfo.fd, size))
450 		ksft_exit_fail_perror("ftruncate()");
451 	p = mmap(BASE_ADDR, size, PROT_READ | PROT_WRITE, MAP_SHARED, finfo.fd,
452 		 0);
453 	if (p != BASE_ADDR)
454 		ksft_exit_fail_perror("mmap()");
455 	return p;
456 }
457 
458 static void shmem_cleanup_area(void *p, unsigned long size)
459 {
460 	munmap(p, size);
461 	close(finfo.fd);
462 }
463 
464 static struct mem_ops __anon_ops = {
465 	.setup_area = &anon_setup_area,
466 	.cleanup_area = &anon_cleanup_area,
467 	.fault = &anon_fault,
468 	.check_huge = &check_huge_anon,
469 	.name = "anon",
470 };
471 
472 static struct mem_ops __read_only_file_ops = {
473 	.setup_area = &file_setup_read_only_area,
474 	.cleanup_area = &file_cleanup_area,
475 	.fault = &file_fault_read,
476 	.check_huge = &check_huge_file,
477 	.name = "file",
478 };
479 
480 static struct mem_ops __read_write_file_read_ops = {
481 	.setup_area = &file_setup_read_write_fs_read_area,
482 	.cleanup_area = &file_cleanup_area,
483 	.fault = &file_fault_read_and_flush,
484 	.check_huge = &check_huge_file,
485 	.name = "file",
486 };
487 
488 static struct mem_ops __read_write_file_write_ops = {
489 	.setup_area = &file_setup_read_write_fs_write_area,
490 	.cleanup_area = &file_cleanup_area,
491 	.fault = &file_fault_write,
492 	.check_huge = &check_huge_file,
493 	.name = "file",
494 };
495 
496 static struct mem_ops __shmem_ops = {
497 	.setup_area = &shmem_setup_area,
498 	.cleanup_area = &shmem_cleanup_area,
499 	.fault = &anon_fault,
500 	.check_huge = &check_huge_file,
501 	.name = "shmem",
502 };
503 
504 static bool is_tmpfs(struct mem_ops *ops)
505 {
506 	return (ops == &__read_only_file_ops ||
507 		ops == &__read_write_file_read_ops ||
508 		ops == &__read_write_file_write_ops) &&
509 	       finfo.type == VMA_SHMEM;
510 }
511 
512 static bool is_anon(struct mem_ops *ops)
513 {
514 	return ops == &__anon_ops;
515 }
516 
517 static void __madvise_collapse(const char *msg, char *p, int nr_hpages,
518 			       struct mem_ops *ops, bool expect)
519 {
520 	struct thp_settings settings = *thp_current_settings();
521 	int ret, i;
522 
523 	ksft_print_msg("%s...", msg);
524 
525 	/*
526 	 * read&write file collapse succeeds for MADV_COLLAPSE because dirty
527 	 * folios are written back after collapse fails for dirty folios and
528 	 * another collapse is attempted.
529 	 */
530 
531 	/*
532 	 * Prevent khugepaged interference and tests that MADV_COLLAPSE
533 	 * ignores /sys/kernel/mm/transparent_hugepage/enabled
534 	 */
535 	settings.thp_enabled = THP_NEVER;
536 	settings.shmem_enabled = SHMEM_NEVER;
537 	for (i = 0; i < NR_ORDERS; i++) {
538 		settings.hugepages[i].enabled = THP_NEVER;
539 		settings.shmem_hugepages[i].enabled = SHMEM_NEVER;
540 	}
541 	thp_push_settings(&settings);
542 
543 	/* Clear VM_NOHUGEPAGE */
544 	madvise(p, nr_hpages * hpage_pmd_size, MADV_HUGEPAGE);
545 	ret = madvise_collapse_retry(p, nr_hpages * hpage_pmd_size);
546 	if (((bool)ret) == expect)
547 		fail("Fail: Bad return value");
548 	else if (!ops->check_huge(p, nr_hpages * hpage_pmd_size, expect ? nr_hpages : 0, hpage_pmd_size))
549 		fail("Fail: check_huge()");
550 	else
551 		success("OK");
552 
553 	thp_pop_settings();
554 }
555 
556 static void madvise_collapse(const char *msg, char *p, int nr_hpages,
557 			     struct mem_ops *ops, bool expect)
558 {
559 	/* Sanity check */
560 	if (!ops->check_huge(p, nr_hpages * hpage_pmd_size, 0, hpage_pmd_size))
561 		ksft_exit_fail_msg("Unexpected huge page\n");
562 	__madvise_collapse(msg, p, nr_hpages, ops, expect);
563 }
564 
565 #define TICK 500000
566 static bool wait_for_scan(const char *msg, char *p, size_t len,
567 		int nr_hpages, int collap_order, struct mem_ops *ops)
568 {
569 	unsigned long hpage_size = page_size << collap_order;
570 	unsigned long bytes = (unsigned long)nr_hpages * hpage_size;
571 	int timeout, full_scans;
572 
573 	/* Half-second ticks: three seconds floor, plus a second per 128M */
574 	timeout = 6 + 2 * (bytes / (128UL << 20));
575 
576 	/* Sanity check */
577 	if (!ops->check_huge(p, len, 0, hpage_size))
578 		ksft_exit_fail_msg("Unexpected huge page\n");
579 
580 	madvise(p, len, MADV_HUGEPAGE);
581 
582 	/* Wait until the second full_scan completed */
583 	full_scans = thp_read_num("khugepaged/full_scans") + 2;
584 
585 	ksft_print_msg("%s...", msg);
586 	while (timeout--) {
587 		if (ops->check_huge(p, len, nr_hpages, hpage_size))
588 			break;
589 		if (thp_read_num("khugepaged/full_scans") >= full_scans)
590 			break;
591 		printf(".");
592 		usleep(TICK);
593 	}
594 
595 	/*
596 	 * The file and shmem tests rely on refaults to install PMD mappings
597 	 * after collapse. MADV_NOHUGEPAGE would prevent those mappings.
598 	 *
599 	 * Apply MADV_NOHUGEPAGE only to anonymous VMAs to prevent khugepaged
600 	 * from unexpectedly collapsing pages during the test.
601 	 */
602 	if (is_anon(ops))
603 		madvise(p, len, MADV_NOHUGEPAGE);
604 
605 	return timeout == -1;
606 }
607 
608 static void khugepaged_collapse(const char *msg, char *p, int nr_hpages,
609 				struct mem_ops *ops, bool expect)
610 {
611 	size_t len = nr_hpages * hpage_pmd_size;
612 
613 	/*
614 	 * read&write file collapse fails since khugepaged does not flush
615 	 * the target dirty folios
616 	 */
617 	if (!is_tmpfs(ops) && ops == &__read_write_file_write_ops)
618 		expect = false;
619 
620 	if (wait_for_scan(msg, p, len, nr_hpages, hpage_pmd_order, ops)) {
621 		if (expect)
622 			fail("Timeout");
623 		else
624 			success("OK");
625 		return;
626 	}
627 
628 	/*
629 	 * For file and shmem memory, khugepaged only retracts pte entries after
630 	 * putting the new hugepage in the page cache. The hugepage must be
631 	 * subsequently refaulted to install the pmd mapping for the mm.
632 	 */
633 	if (ops != &__anon_ops)
634 		ops->fault(p, 0, nr_hpages * hpage_pmd_size);
635 
636 	if (ops->check_huge(p, len, expect ? nr_hpages : 0, hpage_pmd_size))
637 		success("OK");
638 	else
639 		fail("Fail");
640 }
641 
642 static void mthp_khugepaged_collapse(const char *msg, char *p, int nr_hpages,
643 				struct mem_ops *ops, bool expect)
644 {
645 	unsigned long hpage_size = page_size << collapse_order;
646 	struct thp_settings settings = *thp_current_settings();
647 	/* mTHP collpase only allocates PMD sized memory */
648 	size_t len = hpage_pmd_size;
649 
650 	/* Set mTHP setting for mTHP collapse */
651 	if (ops == &__anon_ops) {
652 		settings.thp_enabled = THP_NEVER;
653 		settings.hugepages[collapse_order].enabled = THP_MADVISE;
654 	}
655 
656 	thp_push_settings(&settings);
657 
658 	if (wait_for_scan(msg, p, len, nr_hpages, collapse_order, ops)) {
659 		if (expect)
660 			fail("Timeout");
661 		else
662 			success("OK");
663 
664 		/* Restore THP settings for mTHP collapse. */
665 		thp_pop_settings();
666 		return;
667 	}
668 
669 	/*
670 	 * For file and shmem memory, khugepaged only retracts pte entries after
671 	 * putting the new hugepage in the page cache. The hugepage must be
672 	 * subsequently refaulted to install the pmd mapping for the mm.
673 	 */
674 	if (ops != &__anon_ops)
675 		ops->fault(p, 0, nr_hpages * hpage_size);
676 
677 	if (ops->check_huge(p, len, expect ? nr_hpages : 0, hpage_size))
678 		success("OK");
679 	else
680 		fail("Fail");
681 
682 	/* Restore THP settings for mTHP collapse. */
683 	thp_pop_settings();
684 }
685 
686 static struct collapse_context __khugepaged_context = {
687 	.collapse = &khugepaged_collapse,
688 	.enforce_pte_scan_limits = true,
689 	.name = "khugepaged",
690 };
691 
692 static struct collapse_context __mthp_khugepaged_context = {
693 	.collapse = &mthp_khugepaged_collapse,
694 	.enforce_pte_scan_limits = true,
695 	.name = "mthp_khugepaged",
696 };
697 
698 static struct collapse_context __madvise_context = {
699 	.collapse = &madvise_collapse,
700 	.enforce_pte_scan_limits = false,
701 	.name = "madvise",
702 };
703 
704 static void alloc_at_fault(void)
705 {
706 	struct thp_settings settings = *thp_current_settings();
707 	char *p;
708 
709 	settings.thp_enabled = THP_ALWAYS;
710 	thp_push_settings(&settings);
711 
712 	p = alloc_mapping(1);
713 	*p = 1;
714 	ksft_print_msg("Allocate huge page on fault...");
715 	if (check_huge_anon(p, hpage_pmd_size, 1, hpage_pmd_size))
716 		success("OK");
717 	else
718 		fail("Fail");
719 
720 	thp_pop_settings();
721 
722 	madvise(p, page_size, MADV_DONTNEED);
723 	ksft_print_msg("Split huge PMD on MADV_DONTNEED...");
724 	if (check_huge_anon(p, hpage_pmd_size, 0, hpage_pmd_size))
725 		success("OK");
726 	else
727 		fail("Fail");
728 	munmap(p, hpage_pmd_size);
729 
730 	ksft_test_result_report(exit_status, "allocate on fault and split\n");
731 }
732 
733 static void collapse_full(struct collapse_context *c, struct mem_ops *ops)
734 {
735 	void *p;
736 	int nr_pmds = 4, nr_hpages = 4;
737 	unsigned long size = nr_hpages * hpage_pmd_size;
738 
739 	/* Only try 1 PMD sized range for mTHP collapse. */
740 	if (c == &__mthp_khugepaged_context) {
741 		nr_pmds = 1;
742 		nr_hpages = 1 << (hpage_pmd_order - collapse_order);
743 		size = hpage_pmd_size;
744 	}
745 
746 	p = ops->setup_area(nr_pmds);
747 	ops->fault(p, 0, size);
748 	c->collapse("Collapse multiple fully populated PTE table", p, nr_hpages,
749 		    ops, true);
750 	validate_memory(p, 0, size);
751 	ops->cleanup_area(p, size);
752 
753 	ksft_test_result_report(exit_status, "%s\n", __func__);
754 }
755 
756 static void collapse_empty(struct collapse_context *c, struct mem_ops *ops)
757 {
758 	int nr_hpages = 1;
759 	void *p;
760 
761 	if (c == &__mthp_khugepaged_context)
762 		nr_hpages = 1 << (hpage_pmd_order - collapse_order);
763 
764 	p = ops->setup_area(1);
765 	c->collapse("Do not collapse empty PTE table", p, nr_hpages, ops, false);
766 	ops->cleanup_area(p, hpage_pmd_size);
767 	ksft_test_result_report(exit_status, "%s\n", __func__);
768 }
769 
770 static void collapse_single_mthp(struct collapse_context *c, struct mem_ops *ops)
771 {
772 	unsigned long hpage_size = page_size << collapse_order;
773 	void *p;
774 
775 	p = ops->setup_area(1);
776 	/*
777 	 * Only fault collapse_order sized ranges, and only check 1
778 	 * collapse_order sized huge page.
779 	 */
780 	ops->fault(p, 0, hpage_size);
781 	c->collapse("Collapse PTE table with half PTE entries present",
782 		p, 1, ops, true);
783 	ops->cleanup_area(p, hpage_pmd_size);
784 	ksft_test_result_report(exit_status, "%s\n", __func__);
785 }
786 
787 static void collapse_single_pte_entry(struct collapse_context *c, struct mem_ops *ops)
788 {
789 	void *p;
790 
791 	p = ops->setup_area(1);
792 	ops->fault(p, 0, page_size);
793 	c->collapse("Collapse PTE table with single PTE entry present", p,
794 		    1, ops, true);
795 	ops->cleanup_area(p, hpage_pmd_size);
796 	ksft_test_result_report(exit_status, "%s\n", __func__);
797 }
798 
799 static void collapse_max_ptes_none(struct collapse_context *c, struct mem_ops *ops)
800 {
801 	int max_ptes_none = hpage_pmd_nr / 2;
802 	struct thp_settings settings = *thp_current_settings();
803 	void *p;
804 	int fault_nr_pages = is_anon(ops) ? 1 << anon_order : 1;
805 
806 	settings.khugepaged.max_ptes_none = max_ptes_none;
807 	thp_push_settings(&settings);
808 
809 	p = ops->setup_area(1);
810 
811 	if (is_tmpfs(ops)) {
812 		/* shmem pages always in the page cache */
813 		printf("tmpfs...");
814 		skip("Skip");
815 		goto skip;
816 	}
817 
818 	ops->fault(p, 0, (hpage_pmd_nr - max_ptes_none - fault_nr_pages) * page_size);
819 	c->collapse("Maybe collapse with max_ptes_none exceeded", p, 1,
820 		    ops, !c->enforce_pte_scan_limits);
821 	validate_memory(p, 0, (hpage_pmd_nr - max_ptes_none - fault_nr_pages) * page_size);
822 
823 	if (c->enforce_pte_scan_limits) {
824 		ops->cleanup_area(p, hpage_pmd_size);
825 		p = ops->setup_area(1);
826 
827 		ops->fault(p, 0, (hpage_pmd_nr - max_ptes_none) * page_size);
828 		c->collapse("Collapse with max_ptes_none PTEs empty", p, 1, ops,
829 			    true);
830 		validate_memory(p, 0,
831 				(hpage_pmd_nr - max_ptes_none) * page_size);
832 	}
833 skip:
834 	ops->cleanup_area(p, hpage_pmd_size);
835 	thp_pop_settings();
836 	ksft_test_result_report(exit_status, "%s\n", __func__);
837 }
838 
839 static void collapse_swapin_single_pte(struct collapse_context *c, struct mem_ops *ops)
840 {
841 	void *p;
842 
843 	p = ops->setup_area(1);
844 	ops->fault(p, 0, hpage_pmd_size);
845 
846 	ksft_print_msg("Swapout one page...");
847 	if (swapout_range(p, page_size)) {
848 		success("OK");
849 	} else {
850 		skip("Could not swap out");
851 		goto out;
852 	}
853 
854 	c->collapse("Collapse with swapping in single PTE entry", p, 1, ops,
855 		    true);
856 	validate_memory(p, 0, hpage_pmd_size);
857 out:
858 	ops->cleanup_area(p, hpage_pmd_size);
859 	ksft_test_result_report(exit_status, "%s\n", __func__);
860 }
861 
862 static void collapse_max_ptes_swap(struct collapse_context *c, struct mem_ops *ops)
863 {
864 	int max_ptes_swap = thp_read_num("khugepaged/max_ptes_swap");
865 	void *p;
866 
867 	p = ops->setup_area(1);
868 	ops->fault(p, 0, hpage_pmd_size);
869 
870 	ksft_print_msg("Swapout %d of %d pages...", max_ptes_swap + 1, hpage_pmd_nr);
871 	if (swapout_range(p, (max_ptes_swap + 1) * page_size)) {
872 		success("OK");
873 	} else {
874 		skip("Could not swap out");
875 		goto out;
876 	}
877 
878 	c->collapse("Maybe collapse with max_ptes_swap exceeded", p, 1, ops,
879 		    !c->enforce_pte_scan_limits);
880 	validate_memory(p, 0, hpage_pmd_size);
881 
882 	if (c->enforce_pte_scan_limits) {
883 		ops->fault(p, 0, hpage_pmd_size);
884 		ksft_print_msg("Swapout %d of %d pages...", max_ptes_swap,
885 		       hpage_pmd_nr);
886 		if (swapout_range(p, max_ptes_swap * page_size)) {
887 			success("OK");
888 		} else {
889 			skip("Could not swap out");
890 			goto out;
891 		}
892 
893 		c->collapse("Collapse with max_ptes_swap pages swapped out", p,
894 			    1, ops, true);
895 		validate_memory(p, 0, hpage_pmd_size);
896 	}
897 out:
898 	ops->cleanup_area(p, hpage_pmd_size);
899 	ksft_test_result_report(exit_status, "%s\n", __func__);
900 }
901 
902 static void collapse_single_pte_entry_compound(struct collapse_context *c, struct mem_ops *ops)
903 {
904 	void *p;
905 
906 	p = alloc_hpage(ops);
907 
908 	if (is_tmpfs(ops)) {
909 		/* MADV_DONTNEED won't evict tmpfs pages */
910 		printf("tmpfs...");
911 		skip("Skip");
912 		goto skip;
913 	}
914 
915 	madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE);
916 	ksft_print_msg("Split huge page leaving single PTE mapping compound page...");
917 	madvise(p + page_size, hpage_pmd_size - page_size, MADV_DONTNEED);
918 	if (ops->check_huge(p, hpage_pmd_size, 0, hpage_pmd_size))
919 		success("OK");
920 	else
921 		fail("Fail");
922 
923 	c->collapse("Collapse PTE table with single PTE mapping compound page",
924 		    p, 1, ops, true);
925 	validate_memory(p, 0, page_size);
926 skip:
927 	ops->cleanup_area(p, hpage_pmd_size);
928 	ksft_test_result_report(exit_status, "%s\n", __func__);
929 }
930 
931 static void collapse_full_of_compound(struct collapse_context *c, struct mem_ops *ops)
932 {
933 	void *p;
934 
935 	p = alloc_hpage(ops);
936 	ksft_print_msg("Split huge page leaving single PTE page table full of compound pages...");
937 	madvise(p, page_size, MADV_NOHUGEPAGE);
938 	madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE);
939 	if (ops->check_huge(p, hpage_pmd_size, 0, hpage_pmd_size))
940 		success("OK");
941 	else
942 		fail("Fail");
943 
944 	c->collapse("Collapse PTE table full of compound pages", p, 1, ops,
945 		    true);
946 	validate_memory(p, 0, hpage_pmd_size);
947 	ops->cleanup_area(p, hpage_pmd_size);
948 	ksft_test_result_report(exit_status, "%s\n", __func__);
949 }
950 
951 static void collapse_compound_extreme(struct collapse_context *c, struct mem_ops *ops)
952 {
953 	void *p;
954 	int i;
955 
956 	/*
957 	 * This needs hpage_pmd_nr PMD-order allocations in a row, which the
958 	 * allocator will not supply if the PMD is very large.
959 	 */
960 	if (hpage_pmd_size > (32UL << 20)) {
961 		ksft_test_result_skip("%s: PMD too large for fault-time THP construction\n",
962 				      __func__);
963 		return;
964 	}
965 
966 	p = ops->setup_area(1);
967 	ksft_print_msg("Construct PTE page table full of different PTE-mapped compound pages\n");
968 	for (i = 0; i < hpage_pmd_nr; i++) {
969 		madvise(BASE_ADDR, hpage_pmd_size, MADV_HUGEPAGE);
970 		ops->fault(BASE_ADDR, 0, hpage_pmd_size);
971 		if (!ops->check_huge(BASE_ADDR, hpage_pmd_size, 1, hpage_pmd_size))
972 			ksft_exit_fail_msg("Failed to allocate huge page\n");
973 		madvise(BASE_ADDR, hpage_pmd_size, MADV_NOHUGEPAGE);
974 
975 		p = mremap(BASE_ADDR - i * page_size,
976 				i * page_size + hpage_pmd_size,
977 				(i + 1) * page_size,
978 				MREMAP_MAYMOVE | MREMAP_FIXED,
979 				BASE_ADDR + 2 * hpage_pmd_size);
980 		if (p == MAP_FAILED)
981 			ksft_exit_fail_perror("mremap+unmap");
982 
983 		p = mremap(BASE_ADDR + 2 * hpage_pmd_size,
984 				(i + 1) * page_size,
985 				(i + 1) * page_size + hpage_pmd_size,
986 				MREMAP_MAYMOVE | MREMAP_FIXED,
987 				BASE_ADDR - (i + 1) * page_size);
988 		if (p == MAP_FAILED)
989 			ksft_exit_fail_perror("mremap+alloc");
990 	}
991 
992 	ops->cleanup_area(BASE_ADDR, hpage_pmd_size);
993 	ops->fault(p, 0, hpage_pmd_size);
994 	if (!ops->check_huge(p, hpage_pmd_size, 1, hpage_pmd_size))
995 		success("OK");
996 	else
997 		fail("Fail");
998 
999 	c->collapse("Collapse PTE table full of different compound pages", p, 1,
1000 		    ops, true);
1001 
1002 	validate_memory(p, 0, hpage_pmd_size);
1003 	ops->cleanup_area(p, hpage_pmd_size);
1004 	ksft_test_result_report(exit_status, "%s\n", __func__);
1005 }
1006 
1007 static void collapse_fork(struct collapse_context *c, struct mem_ops *ops)
1008 {
1009 	int wstatus;
1010 	void *p;
1011 
1012 	p = ops->setup_area(1);
1013 
1014 	ksft_print_msg("Allocate small page...");
1015 	ops->fault(p, 0, page_size);
1016 	if (ops->check_huge(p, hpage_pmd_size, 0, hpage_pmd_size))
1017 		success("OK");
1018 	else
1019 		fail("Fail");
1020 
1021 	ksft_print_msg("Share small page over fork()...");
1022 	if (!fork()) {
1023 		/* Do not touch settings on child exit */
1024 		if (ops->check_huge(p, hpage_pmd_size, 0, hpage_pmd_size))
1025 			success("OK");
1026 		else
1027 			fail("Fail");
1028 
1029 		ops->fault(p, page_size, 2 * page_size);
1030 		c->collapse("Collapse PTE table with single page shared with parent process",
1031 			    p, 1, ops, true);
1032 
1033 		validate_memory(p, 0, page_size);
1034 		ops->cleanup_area(p, hpage_pmd_size);
1035 		_exit(exit_status);
1036 	}
1037 
1038 	wait(&wstatus);
1039 	exit_status = WEXITSTATUS(wstatus);
1040 	if (exit_status == KSFT_FAIL)
1041 		goto out;
1042 
1043 	ksft_print_msg("Check if parent still has small page...");
1044 	if (ops->check_huge(p, hpage_pmd_size, 0, hpage_pmd_size))
1045 		success("OK");
1046 	else
1047 		fail("Fail");
1048 	validate_memory(p, 0, page_size);
1049 out:
1050 	ops->cleanup_area(p, hpage_pmd_size);
1051 	ksft_test_result_report(exit_status, "%s\n", __func__);
1052 }
1053 
1054 static void collapse_fork_compound(struct collapse_context *c, struct mem_ops *ops)
1055 {
1056 	int wstatus;
1057 	void *p;
1058 
1059 	p = alloc_hpage(ops);
1060 	ksft_print_msg("Share huge page over fork()...");
1061 	if (!fork()) {
1062 		/* Do not touch settings on child exit */
1063 		if (ops->check_huge(p, hpage_pmd_size, 1, hpage_pmd_size))
1064 			success("OK");
1065 		else
1066 			fail("Fail");
1067 
1068 		ksft_print_msg("Split huge page PMD in child process...");
1069 		madvise(p, page_size, MADV_NOHUGEPAGE);
1070 		madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE);
1071 		if (ops->check_huge(p, hpage_pmd_size, 0, hpage_pmd_size))
1072 			success("OK");
1073 		else
1074 			fail("Fail");
1075 		ops->fault(p, 0, page_size);
1076 
1077 		thp_write_num("khugepaged/max_ptes_shared", hpage_pmd_nr - 1);
1078 		c->collapse("Collapse PTE table full of compound pages in child",
1079 			    p, 1, ops, true);
1080 		thp_write_num("khugepaged/max_ptes_shared",
1081 			  thp_current_settings()->khugepaged.max_ptes_shared);
1082 
1083 		validate_memory(p, 0, hpage_pmd_size);
1084 		ops->cleanup_area(p, hpage_pmd_size);
1085 		_exit(exit_status);
1086 	}
1087 
1088 	wait(&wstatus);
1089 	exit_status = WEXITSTATUS(wstatus);
1090 	if (exit_status == KSFT_FAIL)
1091 		goto out;
1092 
1093 	ksft_print_msg("Check if parent still has huge page...");
1094 	if (ops->check_huge(p, hpage_pmd_size, 1, hpage_pmd_size))
1095 		success("OK");
1096 	else
1097 		fail("Fail");
1098 	validate_memory(p, 0, hpage_pmd_size);
1099 out:
1100 	ops->cleanup_area(p, hpage_pmd_size);
1101 	ksft_test_result_report(exit_status, "%s\n", __func__);
1102 }
1103 
1104 static void collapse_max_ptes_shared(struct collapse_context *c, struct mem_ops *ops)
1105 {
1106 	int max_ptes_shared = thp_read_num("khugepaged/max_ptes_shared");
1107 	int wstatus;
1108 	void *p;
1109 
1110 	p = alloc_hpage(ops);
1111 	ksft_print_msg("Share huge page over fork()...");
1112 	if (!fork()) {
1113 		/* Do not touch settings on child exit */
1114 		if (ops->check_huge(p, hpage_pmd_size, 1, hpage_pmd_size))
1115 			success("OK");
1116 		else
1117 			fail("Fail");
1118 
1119 		ksft_print_msg("Trigger CoW on page %d of %d...",
1120 				hpage_pmd_nr - max_ptes_shared - 1, hpage_pmd_nr);
1121 		ops->fault(p, 0, (hpage_pmd_nr - max_ptes_shared - 1) * page_size);
1122 		if (ops->check_huge(p, hpage_pmd_size, 0, hpage_pmd_size))
1123 			success("OK");
1124 		else
1125 			fail("Fail");
1126 
1127 		c->collapse("Maybe collapse with max_ptes_shared exceeded", p,
1128 			    1, ops, !c->enforce_pte_scan_limits);
1129 
1130 		if (c->enforce_pte_scan_limits) {
1131 			ksft_print_msg("Trigger CoW on page %d of %d...",
1132 			       hpage_pmd_nr - max_ptes_shared, hpage_pmd_nr);
1133 			ops->fault(p, 0, (hpage_pmd_nr - max_ptes_shared) *
1134 				    page_size);
1135 			if (ops->check_huge(p, hpage_pmd_size, 0, hpage_pmd_size))
1136 				success("OK");
1137 			else
1138 				fail("Fail");
1139 
1140 			c->collapse("Collapse with max_ptes_shared PTEs shared",
1141 				    p, 1, ops, true);
1142 		}
1143 
1144 		validate_memory(p, 0, hpage_pmd_size);
1145 		ops->cleanup_area(p, hpage_pmd_size);
1146 		_exit(exit_status);
1147 	}
1148 
1149 	wait(&wstatus);
1150 	exit_status = WEXITSTATUS(wstatus);
1151 	if (exit_status == KSFT_FAIL)
1152 		goto out;
1153 
1154 	ksft_print_msg("Check if parent still has huge page...");
1155 	if (ops->check_huge(p, hpage_pmd_size, 1, hpage_pmd_size))
1156 		success("OK");
1157 	else
1158 		fail("Fail");
1159 	validate_memory(p, 0, hpage_pmd_size);
1160 out:
1161 	ops->cleanup_area(p, hpage_pmd_size);
1162 	ksft_test_result_report(exit_status, "%s\n", __func__);
1163 }
1164 
1165 /*
1166  * The parent writes to the fork-shared range throughout the child's
1167  * collapse.  CoW must keep the two apart: the child sees the pre-fork
1168  * content, the parent only its own writes.
1169  */
1170 static void collapse_fork_cow_race(struct collapse_context *c, struct mem_ops *ops)
1171 {
1172 	const int stride = page_size / sizeof(int);
1173 	int wstatus, child_status, i, n;
1174 	unsigned long shared;
1175 	volatile int *ip;
1176 	pid_t child;
1177 	int sync[2];
1178 	char go = 1;
1179 	void *p;
1180 
1181 	/* At a page per 10 ms, 64 pages spread the writes across the collapse */
1182 	n = 64;
1183 	shared = n * page_size;
1184 
1185 	p = ops->setup_area(1);
1186 	/* Shared prefix, with the pre-fork pattern */
1187 	ops->fault(p, 0, shared);
1188 	if (pipe(sync))
1189 		ksft_exit_fail_perror("pipe()");
1190 
1191 	/* A volatile pointer so the stores are not merged or dropped */
1192 	ip = p;
1193 
1194 	ksft_print_msg("Fork, collapse in the child while the parent rewrites...");
1195 	child = fork();
1196 	if (!child) {
1197 		int collapse_status;
1198 
1199 		close(sync[0]);
1200 		/* Private remainder */
1201 		ops->fault(p, shared, hpage_pmd_size);
1202 		/* Start the parent unsharing, and give it a head start */
1203 		if (write(sync[1], &go, 1) != 1)
1204 			_exit(KSFT_FAIL);
1205 		usleep(5000);
1206 		c->collapse("Collapse a range the parent is writing to",
1207 			    p, 1, ops, true);
1208 		collapse_status = exit_status;
1209 		for (i = 0; i < n; i++)
1210 			if (ip[i * stride] != i + 0xdead0000)
1211 				break;
1212 		if (i == n)
1213 			success("OK");
1214 		else
1215 			fail("Fail: child content");
1216 		/* The content check must not bury a failed collapse */
1217 		if (exit_status != KSFT_FAIL)
1218 			exit_status = collapse_status;
1219 		ops->cleanup_area(p, hpage_pmd_size);
1220 		_exit(exit_status);
1221 	}
1222 
1223 	close(sync[1]);
1224 	if (read(sync[0], &go, 1) != 1)
1225 		ksft_exit_fail_msg("child never reached the collapse\n");
1226 	close(sync[0]);
1227 
1228 	/*
1229 	 * Unshare one page at a time: a burst would break CoW on the whole
1230 	 * range before the collapse starts, leaving nothing shared to collapse.
1231 	 */
1232 	i = 0;
1233 	for (;;) {
1234 		pid_t ret;
1235 
1236 		if (i < n)
1237 			ip[i * stride] = i + 0xbeef0000;
1238 		i++;
1239 		usleep(10 * 1000);
1240 		ret = waitpid(child, &wstatus, WNOHANG);
1241 		if (ret == child)
1242 			break;
1243 		if (ret < 0)
1244 			ksft_exit_fail_perror("waitpid()");
1245 	}
1246 
1247 	/* Finish whatever the paced sweep did not reach */
1248 	for (; i < n; i++)
1249 		ip[i * stride] = i + 0xbeef0000;
1250 	/* A child that died reading the racing pages is a failure, not a zero */
1251 	child_status = WIFEXITED(wstatus) ? WEXITSTATUS(wstatus) : KSFT_FAIL;
1252 
1253 	ksft_print_msg("Check the parent sees only its own writes...");
1254 	for (i = 0; i < n; i++)
1255 		if (ip[i * stride] != i + 0xbeef0000)
1256 			break;
1257 	if (i == n)
1258 		success("OK");
1259 	else
1260 		fail("Fail: parent content");
1261 	ops->cleanup_area(p, hpage_pmd_size);
1262 	/* The parent's check must not bury the child's verdict */
1263 	if (exit_status != KSFT_FAIL)
1264 		exit_status = child_status;
1265 	ksft_test_result_report(exit_status, "%s\n", __func__);
1266 }
1267 
1268 static void madvise_collapse_existing_thps(struct collapse_context *c,
1269 					   struct mem_ops *ops)
1270 {
1271 	void *p;
1272 
1273 	p = ops->setup_area(1);
1274 	ops->fault(p, 0, hpage_pmd_size);
1275 	c->collapse("Collapse fully populated PTE table...", p, 1, ops, true);
1276 	validate_memory(p, 0, hpage_pmd_size);
1277 
1278 	/* c->collapse() will find a hugepage and complain - call directly. */
1279 	__madvise_collapse("Re-collapse PMD-mapped hugepage", p, 1, ops, true);
1280 	validate_memory(p, 0, hpage_pmd_size);
1281 	ops->cleanup_area(p, hpage_pmd_size);
1282 	ksft_test_result_report(exit_status, "%s\n", __func__);
1283 }
1284 
1285 /*
1286  * Test race with khugepaged where page tables have been retracted and
1287  * pmd cleared.
1288  */
1289 static void madvise_retracted_page_tables(struct collapse_context *c,
1290 					  struct mem_ops *ops)
1291 {
1292 	void *p;
1293 	int nr_hpages = 1;
1294 	unsigned long size = nr_hpages * hpage_pmd_size;
1295 
1296 	p = ops->setup_area(nr_hpages);
1297 	ops->fault(p, 0, size);
1298 
1299 	/* Let khugepaged collapse and leave pmd cleared */
1300 	if (wait_for_scan("Collapse and leave PMD cleared", p, size, nr_hpages,
1301 			  hpage_pmd_order, ops)) {
1302 		fail("Timeout");
1303 		return;
1304 	}
1305 	success("OK");
1306 	c->collapse("Install huge PMD from page cache", p, nr_hpages, ops,
1307 		    true);
1308 	validate_memory(p, 0, size);
1309 	ops->cleanup_area(p, size);
1310 	ksft_test_result_report(exit_status, "%s\n", __func__);
1311 }
1312 
1313 /* Smallest order khugepaged will consider for mTHP collapse */
1314 #define MIN_MTHP_ORDER 2
1315 
1316 /* Time budget for one khugepaged pass in the collapse_order_* cases */
1317 #define MTHP_PASS_TIMEOUT_S 30
1318 
1319 static size_t mthp_window_size(void)
1320 {
1321 	return page_size << collapse_order;
1322 }
1323 
1324 static void mthp_push_target_order(void)
1325 {
1326 	struct thp_settings settings = *thp_current_settings();
1327 	int i;
1328 
1329 	/*
1330 	 * Only the target order, and only for madvise: the cases fault their
1331 	 * region first, so the sources stay order 0 whatever -s asked for.
1332 	 */
1333 	settings.thp_enabled = THP_NEVER;
1334 	for (i = 0; i < NR_ORDERS; i++)
1335 		settings.hugepages[i].enabled = THP_NEVER;
1336 	settings.hugepages[collapse_order].enabled = THP_MADVISE;
1337 	thp_push_settings(&settings);
1338 }
1339 
1340 static bool all_windows_at_order(void *p, size_t len)
1341 {
1342 	return is_range_backed_by_order(p, len, collapse_order,
1343 					pagemap_fd, kpageflags_fd);
1344 }
1345 
1346 static bool any_window_at_order(void *p, size_t len)
1347 {
1348 	size_t window = mthp_window_size();
1349 	char *addr = p;
1350 
1351 	for (; len >= window; addr += window, len -= window) {
1352 		if (all_windows_at_order(addr, window))
1353 			return true;
1354 	}
1355 	return false;
1356 }
1357 
1358 static void collapse_order_single_window(struct collapse_context *c,
1359 					 struct mem_ops *ops)
1360 {
1361 	size_t window = mthp_window_size();
1362 	void *p;
1363 
1364 	mthp_push_target_order();
1365 
1366 	p = ops->setup_area(1);
1367 	ops->fault(p, window, 2 * window);
1368 	if (any_window_at_order(p, hpage_pmd_size))
1369 		ksft_exit_fail_msg("Unexpected large folio after fault\n");
1370 
1371 	if (madvise(p, hpage_pmd_size, MADV_HUGEPAGE))
1372 		ksft_exit_fail_perror("madvise(MADV_HUGEPAGE)");
1373 	ksft_print_msg("Collapse one fully populated window...");
1374 	if (!khugepaged_full_pass(MTHP_PASS_TIMEOUT_S))
1375 		fail("Timeout");
1376 	else if (all_windows_at_order(p + window, window) &&
1377 		 !any_window_at_order(p, window) &&
1378 		 !any_window_at_order(p + 2 * window,
1379 				      hpage_pmd_size - 2 * window))
1380 		success("OK");
1381 	else
1382 		fail("Fail");
1383 
1384 	validate_memory(p, window, 2 * window);
1385 	ops->cleanup_area(p, hpage_pmd_size);
1386 	thp_pop_settings();
1387 	ksft_test_result_report(exit_status, "%s\n", __func__);
1388 }
1389 
1390 static void collapse_order_partial_window(struct collapse_context *c,
1391 					  struct mem_ops *ops)
1392 {
1393 	void *p;
1394 
1395 	mthp_push_target_order();
1396 
1397 	p = ops->setup_area(1);
1398 	ops->fault(p, 0, page_size);
1399 	if (any_window_at_order(p, hpage_pmd_size))
1400 		ksft_exit_fail_msg("Unexpected large folio after fault\n");
1401 
1402 	if (madvise(p, hpage_pmd_size, MADV_HUGEPAGE))
1403 		ksft_exit_fail_perror("madvise(MADV_HUGEPAGE)");
1404 	ksft_print_msg("Collapse window with single PTE entry present...");
1405 	if (!khugepaged_full_pass(MTHP_PASS_TIMEOUT_S))
1406 		fail("Timeout");
1407 	else if (all_windows_at_order(p, mthp_window_size()))
1408 		success("OK");
1409 	else
1410 		fail("Fail");
1411 
1412 	validate_memory(p, 0, page_size);
1413 	ops->cleanup_area(p, hpage_pmd_size);
1414 	thp_pop_settings();
1415 	ksft_test_result_report(exit_status, "%s\n", __func__);
1416 }
1417 
1418 static void collapse_order_max_ptes_none(struct collapse_context *c,
1419 					 struct mem_ops *ops)
1420 {
1421 	struct thp_settings settings;
1422 	size_t window = mthp_window_size();
1423 	void *p;
1424 
1425 	mthp_push_target_order();
1426 	settings = *thp_current_settings();
1427 	settings.khugepaged.max_ptes_none = 0;
1428 	thp_push_settings(&settings);
1429 
1430 	p = ops->setup_area(1);
1431 	ops->fault(p, 0, 2 * window - page_size);
1432 	if (any_window_at_order(p, hpage_pmd_size))
1433 		ksft_exit_fail_msg("Unexpected large folio after fault\n");
1434 
1435 	if (madvise(p, hpage_pmd_size, MADV_HUGEPAGE))
1436 		ksft_exit_fail_perror("madvise(MADV_HUGEPAGE)");
1437 	ksft_print_msg("Collapse full window, not the one missing a page...");
1438 	if (!khugepaged_full_pass(MTHP_PASS_TIMEOUT_S))
1439 		fail("Timeout");
1440 	else if (all_windows_at_order(p, window) &&
1441 		 !any_window_at_order(p + window, window))
1442 		success("OK");
1443 	else
1444 		fail("Fail");
1445 
1446 	validate_memory(p, 0, 2 * window - page_size);
1447 	ops->cleanup_area(p, hpage_pmd_size);
1448 	thp_pop_settings();
1449 	thp_pop_settings();
1450 	ksft_test_result_report(exit_status, "%s\n", __func__);
1451 }
1452 
1453 static void collapse_order_mixed_sources(struct collapse_context *c,
1454 					 struct mem_ops *ops)
1455 {
1456 	int source_order = anon_order ? anon_order : MIN_MTHP_ORDER;
1457 	struct thp_settings settings;
1458 	void *p;
1459 
1460 	if (source_order >= collapse_order ||
1461 	    !(thp_supported_orders() & (1UL << source_order))) {
1462 		ksft_test_result_skip("%s: no supported source order below target\n",
1463 				      __func__);
1464 		return;
1465 	}
1466 
1467 	mthp_push_target_order();
1468 
1469 	settings = *thp_current_settings();
1470 	settings.hugepages[source_order].enabled = THP_ALWAYS;
1471 	thp_push_settings(&settings);
1472 	p = ops->setup_area(1);
1473 	ops->fault(p, 0, hpage_pmd_size);
1474 	thp_pop_settings();
1475 
1476 	/*
1477 	 * The allocator can fall back to smaller folios under fragmentation;
1478 	 * having nothing to collapse from is not a failure.
1479 	 */
1480 	if (!is_range_backed_by_order(p, hpage_pmd_size, source_order,
1481 				      pagemap_fd, kpageflags_fd)) {
1482 		ksft_print_msg("No order-%d sources to collapse...", source_order);
1483 		skip("Skip");
1484 		ops->cleanup_area(p, hpage_pmd_size);
1485 		thp_pop_settings();
1486 		ksft_test_result_report(exit_status, "%s\n", __func__);
1487 		return;
1488 	}
1489 
1490 	if (madvise(p, hpage_pmd_size, MADV_HUGEPAGE))
1491 		ksft_exit_fail_perror("madvise(MADV_HUGEPAGE)");
1492 	ksft_print_msg("Collapse region backed by order-%d sources...",
1493 		       source_order);
1494 	if (!khugepaged_full_pass(MTHP_PASS_TIMEOUT_S))
1495 		fail("Timeout");
1496 	else if (all_windows_at_order(p, hpage_pmd_size))
1497 		success("OK");
1498 	else
1499 		fail("Fail");
1500 
1501 	validate_memory(p, 0, hpage_pmd_size);
1502 	ops->cleanup_area(p, hpage_pmd_size);
1503 	thp_pop_settings();
1504 	ksft_test_result_report(exit_status, "%s\n", __func__);
1505 }
1506 
1507 static void usage(void)
1508 {
1509 	fprintf(stderr, "\nUsage: ./khugepaged [OPTIONS] <test type> [dir]\n\n");
1510 	fprintf(stderr, "\t<test type>\t: <context>:<mem_type>\n");
1511 	fprintf(stderr, "\t<context>\t: [all|khugepaged|mthp_khugepaged|madvise]\n");
1512 	fprintf(stderr, "\t<mem_type>\t: [all|anon|file|shmem]\n");
1513 	fprintf(stderr, "\n\t\"file,all\" mem_type requires [dir] argument\n");
1514 	fprintf(stderr, "\n\t\"file,all\" mem_type requires a file system\n");
1515 	fprintf(stderr,	"\twith PMD-sized large folio support\n");
1516 	fprintf(stderr, "\n\tif [dir] is a (sub)directory of a tmpfs mount, tmpfs must be\n");
1517 	fprintf(stderr,	"\tmounted with huge=advise option for khugepaged tests to work\n");
1518 	fprintf(stderr, "\n\tmthp_khugepaged only supports anon mem_type now.\n");
1519 	fprintf(stderr,	"\n\tSupported Options:\n");
1520 	fprintf(stderr,	"\t\t-h: This help message.\n");
1521 	fprintf(stderr,	"\t\t-s: mTHP size, expressed as page order.\n");
1522 	fprintf(stderr,	"\t\t    Defaults to 0. Use this size for anon or shmem allocations.\n");
1523 	fprintf(stderr,	"\t\t-c: collapse order for mTHP collapse, expressed as page order.\n");
1524 	fprintf(stderr,	"\t\t    Defaults to every supported order below the PMD.\n");
1525 	fprintf(stderr,	"\t\t    -s, if set, is the source order for the mixed-source case.\n");
1526 	exit(1);
1527 }
1528 
1529 static void parse_test_type(int argc, char **argv)
1530 {
1531 	bool mthp_context_implied = false;
1532 	int opt;
1533 	char *buf;
1534 	const char *token;
1535 
1536 	while ((opt = getopt(argc, argv, "s:c:h")) != -1) {
1537 		switch (opt) {
1538 		case 's':
1539 			anon_order = atoi(optarg);
1540 			break;
1541 		case 'c':
1542 			collapse_order = atoi(optarg);
1543 			collapse_order_set = true;
1544 			break;
1545 		case 'h':
1546 		default:
1547 			usage();
1548 		}
1549 	}
1550 
1551 	/*
1552 	 * Both orders end up as array indices and shift counts, so neither
1553 	 * can be negative, and a zero collapse order asks for base pages.
1554 	 */
1555 	if (anon_order < 0 || anon_order > hpage_pmd_order)
1556 		ksft_exit_fail_msg("-s takes an order in 0..%d, not %d\n",
1557 				   hpage_pmd_order, anon_order);
1558 	if (collapse_order_set &&
1559 	    (collapse_order <= 0 || collapse_order >= hpage_pmd_order))
1560 		ksft_exit_fail_msg("-c takes an order in 1..%d, not %d\n",
1561 				   hpage_pmd_order - 1, collapse_order);
1562 
1563 	argv += optind;
1564 	argc -= optind;
1565 
1566 	if (argc == 0) {
1567 		/* No arguments: anon under every context */
1568 		khugepaged_context =  &__khugepaged_context;
1569 		mthp_khugepaged_context =  &__mthp_khugepaged_context;
1570 		madvise_context =  &__madvise_context;
1571 		anon_ops = &__anon_ops;
1572 		return;
1573 	}
1574 
1575 	buf = argv[0];
1576 	token = strsep(&buf, ":");
1577 
1578 	if (!strcmp(token, "all")) {
1579 		khugepaged_context =  &__khugepaged_context;
1580 		mthp_khugepaged_context =  &__mthp_khugepaged_context;
1581 		madvise_context =  &__madvise_context;
1582 		/* The mTHP context has only anon cases; let other mem_types drop it */
1583 		mthp_context_implied = true;
1584 	} else if (!strcmp(token, "khugepaged")) {
1585 		khugepaged_context =  &__khugepaged_context;
1586 	} else if (!strcmp(token, "mthp_khugepaged")) {
1587 		mthp_khugepaged_context =  &__mthp_khugepaged_context;
1588 	} else if (!strcmp(token, "madvise")) {
1589 		madvise_context =  &__madvise_context;
1590 	} else {
1591 		usage();
1592 	}
1593 
1594 	if (!buf)
1595 		usage();
1596 
1597 	if (!strcmp(buf, "all")) {
1598 		read_only_file_ops =  &__read_only_file_ops;
1599 		read_write_file_read_ops =  &__read_write_file_read_ops;
1600 		read_write_file_write_ops =  &__read_write_file_write_ops;
1601 		anon_ops = &__anon_ops;
1602 		shmem_ops = &__shmem_ops;
1603 	} else if (!strcmp(buf, "anon")) {
1604 		anon_ops = &__anon_ops;
1605 	} else if (!strcmp(buf, "file")) {
1606 		read_only_file_ops =  &__read_only_file_ops;
1607 		read_write_file_read_ops =  &__read_write_file_read_ops;
1608 		read_write_file_write_ops =  &__read_write_file_write_ops;
1609 		if (mthp_khugepaged_context && !mthp_context_implied)
1610 			usage();
1611 		mthp_khugepaged_context = NULL;
1612 	} else if (!strcmp(buf, "shmem")) {
1613 		shmem_ops = &__shmem_ops;
1614 		if (mthp_khugepaged_context && !mthp_context_implied)
1615 			usage();
1616 		mthp_khugepaged_context = NULL;
1617 	} else {
1618 		usage();
1619 	}
1620 
1621 	if (!read_only_file_ops && !read_write_file_read_ops &&
1622 	    !read_write_file_write_ops)
1623 		return;
1624 
1625 	if (argc != 2)
1626 		usage();
1627 
1628 	get_finfo(argv[1]);
1629 }
1630 
1631 typedef void (*test_fn)(struct collapse_context *c, struct mem_ops *ops);
1632 
1633 struct test_case {
1634 	struct collapse_context *ctx;
1635 	struct mem_ops *ops;
1636 	const char *desc;
1637 	test_fn fn;
1638 	int order;		/* mTHP contexts: the collapse order */
1639 };
1640 
1641 #define MAX_TEST_CASES 256
1642 static struct test_case test_cases[MAX_TEST_CASES];
1643 static int nr_test_cases;
1644 
1645 #define TEST(t, c, o) do {						\
1646 	if (c && o) {							\
1647 		if (nr_test_cases >= MAX_TEST_CASES)			\
1648 			ksft_exit_fail_msg("MAX_TEST_CASES is too small\n"); \
1649 		test_cases[nr_test_cases++] = (struct test_case){	\
1650 			.ctx	= c,					\
1651 			.ops	= o,					\
1652 			.desc	= #t,					\
1653 			.fn	= t,					\
1654 			.order	= collapse_order,			\
1655 		};							\
1656 	}								\
1657 	} while (0)
1658 
1659 int main(int argc, char **argv)
1660 {
1661 	struct thp_settings default_settings = {
1662 		.thp_enabled = THP_MADVISE,
1663 		.thp_defrag = THP_DEFRAG_ALWAYS,
1664 		.shmem_enabled = SHMEM_ADVISE,
1665 		.use_zero_page = 0,
1666 		.khugepaged = {
1667 			.defrag = 1,
1668 			.alloc_sleep_millisecs = 10,
1669 			.scan_sleep_millisecs = 10,
1670 		},
1671 		/*
1672 		 * When testing file-backed memory, the collapse path
1673 		 * looks at how many pages are found in the page cache, not
1674 		 * what pages are mapped. Disable read ahead optimization so
1675 		 * pages don't find their way into the page cache unless
1676 		 * we mem_ops->fault() them in.
1677 		 */
1678 		.read_ahead_kb = 0,
1679 	};
1680 
1681 	ksft_print_header();
1682 
1683 	if (!thp_is_enabled())
1684 		ksft_exit_skip("Transparent Hugepages not available\n");
1685 
1686 	page_size = getpagesize();
1687 	hpage_pmd_size = read_pmd_pagesize();
1688 	if (!hpage_pmd_size)
1689 		ksft_exit_fail_msg("Reading PMD pagesize failed\n");
1690 	hpage_pmd_nr = hpage_pmd_size / page_size;
1691 	hpage_pmd_order = __builtin_ctz(hpage_pmd_nr);
1692 
1693 	parse_test_type(argc, argv);
1694 
1695 	if (mthp_khugepaged_context) {
1696 		unsigned long orders = thp_supported_orders();
1697 
1698 		if (collapse_order_set) {
1699 			if (!(orders & (1UL << collapse_order)))
1700 				ksft_exit_skip("Order %d is not a supported anon THP order\n",
1701 					       collapse_order);
1702 			if (collapse_order <= anon_order)
1703 				ksft_exit_skip("-c %d needs a source order below it, -s says %d\n",
1704 					       collapse_order, anon_order);
1705 			collapse_orders[nr_collapse_orders++] = collapse_order;
1706 		} else {
1707 			/*
1708 			 * Every supported order above the source: -s makes the
1709 			 * fault path hand out folios of that order, so a target
1710 			 * at or below it has nothing to collapse.
1711 			 */
1712 			int first = anon_order + 1;
1713 
1714 			if (first < MIN_MTHP_ORDER)
1715 				first = MIN_MTHP_ORDER;
1716 			for (int i = first; i < hpage_pmd_order; i++) {
1717 				if (orders & (1UL << i))
1718 					collapse_orders[nr_collapse_orders++] = i;
1719 			}
1720 			if (!nr_collapse_orders)
1721 				ksft_print_msg("mTHP cases skipped: no order above the source\n");
1722 		}
1723 	}
1724 
1725 	if (mthp_khugepaged_context) {
1726 		pagemap_fd = open("/proc/self/pagemap", O_RDONLY);
1727 		if (pagemap_fd < 0)
1728 			ksft_exit_fail_perror("open(/proc/self/pagemap)");
1729 		kpageflags_fd = open("/proc/kpageflags", O_RDONLY);
1730 		if (kpageflags_fd < 0)
1731 			ksft_exit_fail_perror("open(/proc/kpageflags)");
1732 	}
1733 
1734 	setbuf(stdout, NULL);
1735 
1736 	/*
1737 	 * Without a PMD-order page cache folio the kernel refuses these
1738 	 * collapses, so there is nothing to test.
1739 	 */
1740 	if (!(thp_file_supported_orders() & (1UL << hpage_pmd_order))) {
1741 		if (shmem_ops) {
1742 			ksft_print_msg("no PMD-order page cache folio: skipping shmem\n");
1743 			shmem_ops = NULL;
1744 		}
1745 		if (read_only_file_ops) {
1746 			ksft_print_msg("no PMD-order page cache folio: skipping file\n");
1747 			read_only_file_ops = NULL;
1748 			read_write_file_read_ops = NULL;
1749 			read_write_file_write_ops = NULL;
1750 		}
1751 		if (!anon_ops && !shmem_ops && !read_only_file_ops)
1752 			ksft_exit_skip("No mem_type left to run\n");
1753 	}
1754 
1755 	default_settings.khugepaged.max_ptes_none = hpage_pmd_nr - 1;
1756 	default_settings.khugepaged.max_ptes_swap = hpage_pmd_nr / 8;
1757 	default_settings.khugepaged.max_ptes_shared = hpage_pmd_nr / 2;
1758 	default_settings.khugepaged.pages_to_scan = hpage_pmd_nr * 8;
1759 	default_settings.hugepages[hpage_pmd_order].enabled = THP_INHERIT;
1760 	default_settings.hugepages[anon_order].enabled = THP_ALWAYS;
1761 	default_settings.shmem_hugepages[hpage_pmd_order].enabled = SHMEM_INHERIT;
1762 	default_settings.shmem_hugepages[anon_order].enabled = SHMEM_ALWAYS;
1763 
1764 	save_settings();
1765 	thp_push_settings(&default_settings);
1766 
1767 	TEST(collapse_full, khugepaged_context, anon_ops);
1768 	TEST(collapse_full, khugepaged_context, read_only_file_ops);
1769 	TEST(collapse_full, khugepaged_context, read_write_file_read_ops);
1770 	TEST(collapse_full, khugepaged_context, read_write_file_write_ops);
1771 	TEST(collapse_full, khugepaged_context, shmem_ops);
1772 	for (int i = 0; i < nr_collapse_orders; i++) {
1773 		collapse_order = collapse_orders[i];
1774 		TEST(collapse_full, mthp_khugepaged_context, anon_ops);
1775 		TEST(collapse_empty, mthp_khugepaged_context, anon_ops);
1776 		TEST(collapse_single_mthp, mthp_khugepaged_context, anon_ops);
1777 		TEST(collapse_order_single_window, mthp_khugepaged_context, anon_ops);
1778 		TEST(collapse_order_partial_window, mthp_khugepaged_context, anon_ops);
1779 		TEST(collapse_order_max_ptes_none, mthp_khugepaged_context, anon_ops);
1780 		TEST(collapse_order_mixed_sources, mthp_khugepaged_context, anon_ops);
1781 	}
1782 
1783 	TEST(collapse_full, madvise_context, anon_ops);
1784 	TEST(collapse_full, madvise_context, read_only_file_ops);
1785 	TEST(collapse_full, madvise_context, read_write_file_read_ops);
1786 	TEST(collapse_full, madvise_context, read_write_file_write_ops);
1787 	TEST(collapse_full, madvise_context, shmem_ops);
1788 
1789 	TEST(collapse_empty, khugepaged_context, anon_ops);
1790 	TEST(collapse_empty, madvise_context, anon_ops);
1791 
1792 	TEST(collapse_single_pte_entry, khugepaged_context, anon_ops);
1793 	TEST(collapse_single_pte_entry, khugepaged_context, read_only_file_ops);
1794 	TEST(collapse_single_pte_entry, khugepaged_context, read_write_file_read_ops);
1795 	TEST(collapse_single_pte_entry, khugepaged_context, read_write_file_write_ops);
1796 	TEST(collapse_single_pte_entry, khugepaged_context, shmem_ops);
1797 	TEST(collapse_single_pte_entry, madvise_context, anon_ops);
1798 	TEST(collapse_single_pte_entry, madvise_context, read_only_file_ops);
1799 	TEST(collapse_single_pte_entry, madvise_context, read_write_file_read_ops);
1800 	TEST(collapse_single_pte_entry, madvise_context, read_write_file_write_ops);
1801 	TEST(collapse_single_pte_entry, madvise_context, shmem_ops);
1802 
1803 	TEST(collapse_max_ptes_none, khugepaged_context, anon_ops);
1804 	TEST(collapse_max_ptes_none, khugepaged_context, read_only_file_ops);
1805 	TEST(collapse_max_ptes_none, khugepaged_context, read_write_file_read_ops);
1806 	TEST(collapse_max_ptes_none, khugepaged_context, read_write_file_write_ops);
1807 	TEST(collapse_max_ptes_none, madvise_context, anon_ops);
1808 	TEST(collapse_max_ptes_none, madvise_context, read_only_file_ops);
1809 	TEST(collapse_max_ptes_none, madvise_context, read_write_file_read_ops);
1810 	TEST(collapse_max_ptes_none, madvise_context, read_write_file_write_ops);
1811 
1812 	TEST(collapse_single_pte_entry_compound, khugepaged_context, anon_ops);
1813 	TEST(collapse_single_pte_entry_compound, khugepaged_context, read_only_file_ops);
1814 	TEST(collapse_single_pte_entry_compound, khugepaged_context, read_write_file_read_ops);
1815 	TEST(collapse_single_pte_entry_compound, madvise_context, anon_ops);
1816 	TEST(collapse_single_pte_entry_compound, madvise_context, read_only_file_ops);
1817 	TEST(collapse_single_pte_entry_compound, madvise_context, read_write_file_read_ops);
1818 
1819 	TEST(collapse_full_of_compound, khugepaged_context, anon_ops);
1820 	TEST(collapse_full_of_compound, khugepaged_context, read_only_file_ops);
1821 	TEST(collapse_full_of_compound, khugepaged_context, read_write_file_read_ops);
1822 	TEST(collapse_full_of_compound, khugepaged_context, shmem_ops);
1823 	TEST(collapse_full_of_compound, madvise_context, anon_ops);
1824 	TEST(collapse_full_of_compound, madvise_context, read_only_file_ops);
1825 	TEST(collapse_full_of_compound, madvise_context, read_write_file_read_ops);
1826 	TEST(collapse_full_of_compound, madvise_context, shmem_ops);
1827 
1828 	TEST(collapse_compound_extreme, khugepaged_context, anon_ops);
1829 	TEST(collapse_compound_extreme, madvise_context, anon_ops);
1830 
1831 	TEST(collapse_swapin_single_pte, khugepaged_context, anon_ops);
1832 	TEST(collapse_swapin_single_pte, madvise_context, anon_ops);
1833 
1834 	TEST(collapse_max_ptes_swap, khugepaged_context, anon_ops);
1835 	TEST(collapse_max_ptes_swap, madvise_context, anon_ops);
1836 
1837 	TEST(collapse_fork, khugepaged_context, anon_ops);
1838 	TEST(collapse_fork, madvise_context, anon_ops);
1839 
1840 	TEST(collapse_fork_compound, khugepaged_context, anon_ops);
1841 	TEST(collapse_fork_compound, madvise_context, anon_ops);
1842 
1843 	TEST(collapse_max_ptes_shared, khugepaged_context, anon_ops);
1844 	TEST(collapse_max_ptes_shared, madvise_context, anon_ops);
1845 
1846 	TEST(collapse_fork_cow_race, khugepaged_context, anon_ops);
1847 	TEST(collapse_fork_cow_race, madvise_context, anon_ops);
1848 
1849 	TEST(madvise_collapse_existing_thps, madvise_context, anon_ops);
1850 	TEST(madvise_collapse_existing_thps, madvise_context, read_only_file_ops);
1851 	TEST(madvise_collapse_existing_thps, madvise_context, read_write_file_read_ops);
1852 	TEST(madvise_collapse_existing_thps, madvise_context, shmem_ops);
1853 
1854 	TEST(madvise_retracted_page_tables, madvise_context, read_only_file_ops);
1855 	TEST(madvise_retracted_page_tables, madvise_context, read_write_file_read_ops);
1856 	TEST(madvise_retracted_page_tables, madvise_context, shmem_ops);
1857 
1858 	ksft_set_plan(nr_test_cases + 1);
1859 
1860 	alloc_at_fault();
1861 	for (int i = 0; i < nr_test_cases; i++) {
1862 		struct test_case *t = &test_cases[i];
1863 
1864 		if (t->ctx == &__mthp_khugepaged_context) {
1865 			collapse_order = t->order;
1866 			ksft_print_msg("\n# Run test: %s (%s:%s, order %d)\n",
1867 				       t->desc, t->ctx->name, t->ops->name,
1868 				       t->order);
1869 		} else {
1870 			ksft_print_msg("\n# Run test: %s (%s:%s)\n", t->desc,
1871 				       t->ctx->name, t->ops->name);
1872 		}
1873 		t->fn(t->ctx, t->ops);
1874 	}
1875 
1876 	ksft_finished();
1877 }
1878