1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * A test of splitting PMD THPs and PTE-mapped THPs from a specified virtual
4 * address range in a process via <debugfs>/split_huge_pages interface.
5 */
6
7 #define _GNU_SOURCE
8 #include <assert.h>
9 #include <stdio.h>
10 #include <stdlib.h>
11 #include <stdarg.h>
12 #include <unistd.h>
13 #include <inttypes.h>
14 #include <string.h>
15 #include <fcntl.h>
16 #include <sys/mman.h>
17 #include <sys/mount.h>
18 #include <sys/param.h>
19 #include <malloc.h>
20 #include <stdbool.h>
21 #include <time.h>
22 #include "vm_util.h"
23 #include "kselftest.h"
24 #include "hugepage_settings.h"
25
26 uint64_t pagesize;
27 unsigned int pageshift;
28 uint64_t pmd_pagesize;
29 unsigned int pmd_order;
30 int *expected_orders;
31
32 #define SPLIT_DEBUGFS "/sys/kernel/debug/split_huge_pages"
33 #define SMAP_PATH "/proc/self/smaps"
34 #define INPUT_MAX 80
35
36 #define PID_FMT "%d,0x%lx,0x%lx,%d"
37 #define PID_FMT_OFFSET "%d,0x%lx,0x%lx,%d,%d"
38 #define PATH_FMT "%s,0x%lx,0x%lx,%d"
39
40 const char *pagemap_proc = "/proc/self/pagemap";
41 const char *kpageflags_proc = "/proc/kpageflags";
42 int pagemap_fd;
43 int kpageflags_fd;
44
is_backed_by_folio(char * vaddr,int order,int pagemap_fd,int kpageflags_fd)45 static bool is_backed_by_folio(char *vaddr, int order, int pagemap_fd,
46 int kpageflags_fd)
47 {
48 const uint64_t folio_head_flags = KPF_THP | KPF_COMPOUND_HEAD;
49 const uint64_t folio_tail_flags = KPF_THP | KPF_COMPOUND_TAIL;
50 const unsigned long nr_pages = 1UL << order;
51 unsigned long pfn_head;
52 uint64_t pfn_flags;
53 unsigned long pfn;
54 unsigned long i;
55
56 pfn = pagemap_get_pfn(pagemap_fd, vaddr);
57
58 /* non present page */
59 if (pfn == -1UL)
60 return false;
61
62 if (pageflags_get(pfn, kpageflags_fd, &pfn_flags))
63 goto fail;
64
65 /* check for order-0 pages */
66 if (!order) {
67 if (pfn_flags & (folio_head_flags | folio_tail_flags))
68 return false;
69 return true;
70 }
71
72 /* non THP folio */
73 if (!(pfn_flags & KPF_THP))
74 return false;
75
76 pfn_head = pfn & ~(nr_pages - 1);
77
78 if (pageflags_get(pfn_head, kpageflags_fd, &pfn_flags))
79 goto fail;
80
81 /* head PFN has no compound_head flag set */
82 if ((pfn_flags & folio_head_flags) != folio_head_flags)
83 return false;
84
85 /* check all tail PFN flags */
86 for (i = 1; i < nr_pages; i++) {
87 if (pageflags_get(pfn_head + i, kpageflags_fd, &pfn_flags))
88 goto fail;
89 if ((pfn_flags & folio_tail_flags) != folio_tail_flags)
90 return false;
91 }
92
93 /*
94 * check the PFN after this folio, but if its flags cannot be obtained,
95 * assume this folio has the expected order
96 */
97 if (pageflags_get(pfn_head + nr_pages, kpageflags_fd, &pfn_flags))
98 return true;
99
100 /* If we find another tail page, then the folio is larger. */
101 return (pfn_flags & folio_tail_flags) != folio_tail_flags;
102 fail:
103 ksft_exit_fail_msg("Failed to get folio info\n");
104 return false;
105 }
106
check_after_split_folio_orders(char * vaddr_start,size_t len,int pagemap_fd,int kpageflags_fd,int orders[],int nr_orders)107 static int check_after_split_folio_orders(char *vaddr_start, size_t len,
108 int pagemap_fd, int kpageflags_fd, int orders[], int nr_orders)
109 {
110 int *vaddr_orders;
111 int status;
112 int i;
113
114 vaddr_orders = (int *)malloc(sizeof(int) * nr_orders);
115
116 if (!vaddr_orders)
117 ksft_exit_fail_msg("Cannot allocate memory for vaddr_orders");
118
119 memset(vaddr_orders, 0, sizeof(int) * nr_orders);
120 status = gather_folio_orders(vaddr_start, len, pagemap_fd,
121 kpageflags_fd, vaddr_orders, nr_orders);
122 if (status)
123 ksft_exit_fail_msg("gather folio info failed\n");
124
125 for (i = 0; i < nr_orders; i++)
126 if (vaddr_orders[i] != orders[i]) {
127 ksft_print_msg("order %d: expected: %d got %d\n", i,
128 orders[i], vaddr_orders[i]);
129 status = -1;
130 }
131
132 free(vaddr_orders);
133 return status;
134 }
135
write_debugfs(const char * fmt,...)136 static void write_debugfs(const char *fmt, ...)
137 {
138 char input[INPUT_MAX];
139 int ret;
140 va_list argp;
141
142 va_start(argp, fmt);
143 ret = vsnprintf(input, INPUT_MAX, fmt, argp);
144 va_end(argp);
145
146 if (ret >= INPUT_MAX)
147 ksft_exit_fail_msg("%s: Debugfs input is too long\n", __func__);
148
149 write_file(SPLIT_DEBUGFS, input, ret + 1);
150 }
151
allocate_zero_filled_hugepage(size_t len)152 static char *allocate_zero_filled_hugepage(size_t len)
153 {
154 char *result;
155 size_t i;
156
157 result = memalign(pmd_pagesize, len);
158 if (!result) {
159 printf("Fail to allocate memory\n");
160 exit(EXIT_FAILURE);
161 }
162
163 madvise(result, len, MADV_HUGEPAGE);
164
165 for (i = 0; i < len; i++)
166 result[i] = (char)0;
167
168 return result;
169 }
170
verify_rss_anon_split_huge_page_all_zeroes(char * one_page,int nr_hpages,size_t len)171 static void verify_rss_anon_split_huge_page_all_zeroes(char *one_page, int nr_hpages, size_t len)
172 {
173 unsigned long rss_anon_before, rss_anon_after;
174 size_t i;
175
176 if (!check_huge_anon(one_page, nr_hpages * pmd_pagesize, nr_hpages, pmd_pagesize))
177 ksft_exit_fail_msg("No THP is allocated\n");
178
179 rss_anon_before = rss_anon();
180 if (!rss_anon_before)
181 ksft_exit_fail_msg("No RssAnon is allocated before split\n");
182
183 /* split all THPs */
184 write_debugfs(PID_FMT, getpid(), (uint64_t)one_page,
185 (uint64_t)one_page + len, 0);
186
187 for (i = 0; i < len; i++)
188 if (one_page[i] != (char)0)
189 ksft_exit_fail_msg("%ld byte corrupted\n", i);
190
191 if (!check_huge_anon(one_page, nr_hpages * pmd_pagesize, 0, pmd_pagesize))
192 ksft_exit_fail_msg("Still AnonHugePages not split\n");
193
194 rss_anon_after = rss_anon();
195 if (rss_anon_after >= rss_anon_before)
196 ksft_exit_fail_msg("Incorrect RssAnon value. Before: %ld After: %ld\n",
197 rss_anon_before, rss_anon_after);
198 }
199
split_pmd_zero_pages(void)200 static void split_pmd_zero_pages(void)
201 {
202 char *one_page;
203 int nr_hpages = 4;
204 size_t len = nr_hpages * pmd_pagesize;
205
206 one_page = allocate_zero_filled_hugepage(len);
207 verify_rss_anon_split_huge_page_all_zeroes(one_page, nr_hpages, len);
208 ksft_test_result_pass("Split zero filled huge pages successful\n");
209 free(one_page);
210 }
211
split_pmd_thp_to_order(int order)212 static void split_pmd_thp_to_order(int order)
213 {
214 char *one_page;
215 size_t len = 4 * pmd_pagesize;
216 size_t i;
217
218 one_page = memalign(pmd_pagesize, len);
219 if (!one_page)
220 ksft_exit_fail_msg("Fail to allocate memory: %s\n", strerror(errno));
221
222 madvise(one_page, len, MADV_HUGEPAGE);
223
224 for (i = 0; i < len; i++)
225 one_page[i] = (char)i;
226
227 if (!check_huge_anon(one_page, 4 * pmd_pagesize, 4, pmd_pagesize))
228 ksft_exit_fail_msg("No THP is allocated\n");
229
230 /* split all THPs */
231 write_debugfs(PID_FMT, getpid(), (uint64_t)one_page,
232 (uint64_t)one_page + len, order);
233
234 for (i = 0; i < len; i++)
235 if (one_page[i] != (char)i)
236 ksft_exit_fail_msg("%ld byte corrupted\n", i);
237
238 memset(expected_orders, 0, sizeof(int) * (pmd_order + 1));
239 expected_orders[order] = 4 << (pmd_order - order);
240
241 if (check_after_split_folio_orders(one_page, len, pagemap_fd,
242 kpageflags_fd, expected_orders,
243 (pmd_order + 1)))
244 ksft_exit_fail_msg("Unexpected THP split\n");
245
246 if (!check_huge_anon(one_page, 4 * pmd_pagesize, 0, pmd_pagesize))
247 ksft_exit_fail_msg("Still AnonHugePages not split\n");
248
249 ksft_test_result_pass("Split huge pages to order %d successful\n", order);
250 free(one_page);
251 }
252
split_pte_mapped_thp(void)253 static void split_pte_mapped_thp(void)
254 {
255 const size_t nr_thps = 4;
256 const size_t thp_area_size = nr_thps * pmd_pagesize;
257 const size_t page_area_size = nr_thps * pagesize;
258 char *thp_area, *tmp, *page_area = MAP_FAILED;
259 size_t i;
260
261 thp_area = mmap((void *)(1UL << 30), thp_area_size, PROT_READ | PROT_WRITE,
262 MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
263 if (thp_area == MAP_FAILED) {
264 ksft_test_result_fail("Fail to allocate memory: %s\n", strerror(errno));
265 return;
266 }
267
268 madvise(thp_area, thp_area_size, MADV_HUGEPAGE);
269
270 for (i = 0; i < thp_area_size; i++)
271 thp_area[i] = (char)i;
272
273 if (!check_huge_anon(thp_area, nr_thps * pmd_pagesize, nr_thps, pmd_pagesize)) {
274 ksft_test_result_skip("Not all THPs allocated\n");
275 goto out;
276 }
277
278 /*
279 * To challenge spitting code, we will mremap a single page of each
280 * THP (page[i] of thp[i]) in the thp_area into page_area. This will
281 * replace the PMD mappings in the thp_area by PTE mappings first,
282 * but leaving the THP unsplit, to then create a page-sized hole in
283 * the thp_area.
284 * We will then manually trigger splitting of all THPs through the
285 * single mremap'ed pages of each THP in the page_area.
286 */
287 page_area = mmap(NULL, page_area_size, PROT_READ | PROT_WRITE,
288 MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
289 if (page_area == MAP_FAILED) {
290 ksft_test_result_fail("Fail to allocate memory: %s\n", strerror(errno));
291 goto out;
292 }
293
294 for (i = 0; i < nr_thps; i++) {
295 tmp = mremap(thp_area + pmd_pagesize * i + pagesize * i,
296 pagesize, pagesize, MREMAP_MAYMOVE|MREMAP_FIXED,
297 page_area + pagesize * i);
298 if (tmp != MAP_FAILED)
299 continue;
300 ksft_test_result_fail("mremap failed: %s\n", strerror(errno));
301 goto out;
302 }
303
304 /*
305 * Verify that our THPs were not split yet. Note that
306 * check_huge_anon() cannot be used as it checks for PMD mappings.
307 */
308 for (i = 0; i < nr_thps; i++) {
309 if (is_backed_by_folio(page_area + i * pagesize, pmd_order,
310 pagemap_fd, kpageflags_fd))
311 continue;
312 ksft_test_result_fail("THP %zu missing after mremap\n", i);
313 goto out;
314 }
315
316 /* Split all THPs through the remapped pages. */
317 write_debugfs(PID_FMT, getpid(), (uint64_t)page_area,
318 (uint64_t)page_area + page_area_size, 0);
319
320 /* Corruption during mremap or split? */
321 for (i = 0; i < page_area_size; i++) {
322 if (page_area[i] == (char)i)
323 continue;
324 ksft_test_result_fail("%zu byte corrupted\n", i);
325 goto out;
326 }
327
328 /* Split failed? */
329 for (i = 0; i < nr_thps; i++) {
330 if (is_backed_by_folio(page_area + i * pagesize, 0,
331 pagemap_fd, kpageflags_fd))
332 continue;
333 ksft_test_result_fail("THP %zu not split\n", i);
334 }
335
336 ksft_test_result_pass("Split PTE-mapped huge pages successful\n");
337 out:
338 munmap(thp_area, thp_area_size);
339 if (page_area != MAP_FAILED)
340 munmap(page_area, page_area_size);
341 }
342
split_file_backed_thp(int order)343 static void split_file_backed_thp(int order)
344 {
345 int status;
346 int fd;
347 char tmpfs_template[] = "/tmp/thp_split_XXXXXX";
348 const char *tmpfs_loc = mkdtemp(tmpfs_template);
349 char testfile[INPUT_MAX];
350 unsigned long size = 2 * pmd_pagesize;
351 char opts[64];
352 ssize_t num_written, num_read;
353 char *file_buf1 = NULL, *file_buf2 = NULL;
354 uint64_t pgoff_start = 0, pgoff_end = 1024;
355 int i;
356
357 ksft_print_msg("Please enable pr_debug in split_huge_pages_in_file() for more info.\n");
358
359 if (!tmpfs_loc)
360 ksft_exit_fail_msg("mkdtemp failed\n");
361
362 file_buf1 = (char *)malloc(pmd_pagesize);
363 file_buf2 = (char *)malloc(pmd_pagesize);
364
365 if (!file_buf1 || !file_buf2) {
366 ksft_print_msg("cannot allocate file buffers\n");
367 goto out;
368 }
369
370 for (i = 0; i < pmd_pagesize; i++)
371 file_buf1[i] = (char)i;
372 memset(file_buf2, 0, pmd_pagesize);
373
374 snprintf(opts, sizeof(opts), "huge=always,size=%lu", size);
375 status = mount("tmpfs", tmpfs_loc, "tmpfs", 0, opts);
376
377 if (status) {
378 ksft_print_msg("Unable to create a tmpfs for testing\n");
379 goto out;
380 }
381
382 status = snprintf(testfile, INPUT_MAX, "%s/thp_file", tmpfs_loc);
383 if (status >= INPUT_MAX) {
384 ksft_print_msg("Fail to create file-backed THP split testing file\n");
385 goto cleanup;
386 }
387
388 fd = open(testfile, O_CREAT|O_RDWR, 0664);
389 if (fd == -1) {
390 ksft_perror("Cannot open testing file");
391 goto cleanup;
392 }
393
394 /* write pmd size data to the file, so a file-backed THP can be allocated */
395 num_written = write(fd, file_buf1, pmd_pagesize);
396
397 if (num_written == -1 || num_written != pmd_pagesize) {
398 ksft_perror("Failed to write data to testing file");
399 goto close_file;
400 }
401
402 /* split the file-backed THP */
403 write_debugfs(PATH_FMT, testfile, pgoff_start, pgoff_end, order);
404
405 /* check file content after split */
406 status = lseek(fd, 0, SEEK_SET);
407 if (status == -1) {
408 ksft_perror("Cannot lseek file");
409 goto close_file;
410 }
411
412 num_read = read(fd, file_buf2, num_written);
413 if (num_read == -1 || num_read != num_written) {
414 ksft_perror("Cannot read file content back");
415 goto close_file;
416 }
417
418 if (strncmp(file_buf1, file_buf2, pmd_pagesize) != 0) {
419 ksft_print_msg("File content changed\n");
420 goto close_file;
421 }
422
423 close(fd);
424 status = unlink(testfile);
425 if (status) {
426 ksft_perror("Cannot remove testing file");
427 goto cleanup;
428 }
429
430 status = umount(tmpfs_loc);
431 if (status) {
432 ksft_print_msg("Unable to umount %s\n", tmpfs_loc);
433 goto out;
434 }
435
436 free(file_buf1);
437 free(file_buf2);
438
439 status = rmdir(tmpfs_loc);
440 if (status)
441 ksft_exit_fail_msg("cannot remove tmp dir: %s\n", strerror(errno));
442
443 ksft_print_msg("Please check dmesg for more information\n");
444 ksft_test_result_pass("File-backed THP split to order %d test done\n", order);
445 return;
446
447 close_file:
448 close(fd);
449 cleanup:
450 umount(tmpfs_loc);
451 out:
452 free(file_buf1);
453 free(file_buf2);
454 rmdir(tmpfs_loc);
455 ksft_exit_fail_msg("Error occurred\n");
456 }
457
prepare_thp_fs(const char * xfs_path,char * thp_fs_template,const char ** thp_fs_loc)458 static bool prepare_thp_fs(const char *xfs_path, char *thp_fs_template,
459 const char **thp_fs_loc)
460 {
461 if (xfs_path) {
462 *thp_fs_loc = xfs_path;
463 return false;
464 }
465
466 *thp_fs_loc = mkdtemp(thp_fs_template);
467
468 if (!*thp_fs_loc)
469 ksft_exit_fail_msg("cannot create temp folder\n");
470
471 return true;
472 }
473
cleanup_thp_fs(const char * thp_fs_loc,bool created_tmp)474 static void cleanup_thp_fs(const char *thp_fs_loc, bool created_tmp)
475 {
476 int status;
477
478 if (!created_tmp)
479 return;
480
481 status = rmdir(thp_fs_loc);
482 if (status)
483 ksft_exit_fail_msg("cannot remove tmp dir: %s\n",
484 strerror(errno));
485 }
486
create_pagecache_thp_and_fd(const char * testfile,size_t fd_size,int * fd,char ** addr)487 static int create_pagecache_thp_and_fd(const char *testfile, size_t fd_size,
488 int *fd, char **addr)
489 {
490 size_t i;
491 unsigned char buf[1024];
492
493 srand(time(NULL));
494
495 *fd = open(testfile, O_CREAT | O_RDWR, 0664);
496 if (*fd == -1)
497 ksft_exit_fail_msg("Failed to create a file at %s\n", testfile);
498
499 assert(fd_size % sizeof(buf) == 0);
500 for (i = 0; i < sizeof(buf); i++)
501 buf[i] = (unsigned char)i;
502 for (i = 0; i < fd_size; i += sizeof(buf)) {
503 if (write(*fd, buf, sizeof(buf)) != sizeof(buf)) {
504 ksft_perror("write testfile");
505 close(*fd);
506 goto err_out_unlink;
507 }
508 }
509 close(*fd);
510 sync();
511 *fd = open("/proc/sys/vm/drop_caches", O_WRONLY);
512 if (*fd == -1) {
513 ksft_perror("open drop_caches");
514 goto err_out_unlink;
515 }
516 if (write(*fd, "3", 1) != 1) {
517 ksft_perror("write to drop_caches");
518 goto err_out_close;
519 }
520 close(*fd);
521
522 *fd = open(testfile, O_RDWR);
523 if (*fd == -1) {
524 ksft_perror("Failed to open testfile\n");
525 goto err_out_unlink;
526 }
527
528 *addr = mmap(NULL, fd_size, PROT_READ|PROT_WRITE, MAP_SHARED, *fd, 0);
529 if (*addr == (char *)-1) {
530 ksft_perror("cannot mmap");
531 goto err_out_close;
532 }
533 madvise(*addr, fd_size, MADV_HUGEPAGE);
534
535 force_read_pages(*addr, fd_size / pmd_pagesize, pmd_pagesize);
536
537 if (!check_huge_file(*addr, fd_size, fd_size / pmd_pagesize, pmd_pagesize)) {
538 ksft_print_msg("No large pagecache folio generated, please provide a filesystem supporting large folio\n");
539 munmap(*addr, fd_size);
540 close(*fd);
541 unlink(testfile);
542 ksft_test_result_skip("Pagecache folio split skipped\n");
543 return -2;
544 }
545 return 0;
546 err_out_close:
547 close(*fd);
548 err_out_unlink:
549 unlink(testfile);
550 ksft_exit_fail_msg("Failed to create large pagecache folios\n");
551 return -1;
552 }
553
split_thp_in_pagecache_to_order_at(size_t fd_size,const char * fs_loc,int order,int offset)554 static void split_thp_in_pagecache_to_order_at(size_t fd_size,
555 const char *fs_loc, int order, int offset)
556 {
557 int fd;
558 char *split_addr;
559 char *addr;
560 size_t i;
561 char testfile[INPUT_MAX];
562 int err = 0;
563
564 err = snprintf(testfile, INPUT_MAX, "%s/test", fs_loc);
565
566 if (err < 0)
567 ksft_exit_fail_msg("cannot generate right test file name\n");
568
569 err = create_pagecache_thp_and_fd(testfile, fd_size, &fd, &addr);
570 if (err)
571 return;
572
573 err = 0;
574
575 memset(expected_orders, 0, sizeof(int) * (pmd_order + 1));
576 /*
577 * use [split_addr, split_addr + pagesize) range to split THPs, since
578 * the debugfs function always split a range with pagesize step and
579 * providing a full [addr, addr + fd_size) range can trigger multiple
580 * splits, complicating after-split result checking.
581 */
582 if (offset == -1) {
583 for (split_addr = addr; split_addr < addr + fd_size; split_addr += pmd_pagesize)
584 write_debugfs(PID_FMT, getpid(), (uint64_t)split_addr,
585 (uint64_t)split_addr + pagesize, order);
586
587 expected_orders[order] = fd_size / (pagesize << order);
588 } else {
589 int times = fd_size / pmd_pagesize;
590
591 for (split_addr = addr; split_addr < addr + fd_size; split_addr += pmd_pagesize)
592 write_debugfs(PID_FMT_OFFSET, getpid(), (uint64_t)split_addr,
593 (uint64_t)split_addr + pagesize, order, offset);
594
595 for (i = order + 1; i < pmd_order; i++)
596 expected_orders[i] = times;
597 expected_orders[order] = 2 * times;
598 }
599
600 for (i = 0; i < fd_size; i++)
601 if (*(addr + i) != (char)i) {
602 ksft_print_msg("%lu byte corrupted in the file\n", i);
603 err = EXIT_FAILURE;
604 goto out;
605 }
606
607 if (check_after_split_folio_orders(addr, fd_size, pagemap_fd,
608 kpageflags_fd, expected_orders,
609 (pmd_order + 1))) {
610 ksft_print_msg("Unexpected THP split\n");
611 err = 1;
612 goto out;
613 }
614
615 if (!check_huge_file(addr, fd_size, 0, pmd_pagesize)) {
616 ksft_print_msg("Still FilePmdMapped not split\n");
617 err = EXIT_FAILURE;
618 goto out;
619 }
620
621 out:
622 munmap(addr, fd_size);
623 close(fd);
624 unlink(testfile);
625 if (offset == -1) {
626 if (err)
627 ksft_exit_fail_msg("Split PMD-mapped pagecache folio to order %d failed\n", order);
628 ksft_test_result_pass("Split PMD-mapped pagecache folio to order %d passed\n", order);
629 } else {
630 if (err)
631 ksft_exit_fail_msg("Split PMD-mapped pagecache folio to order %d at in-folio offset %d failed\n", order, offset);
632 ksft_test_result_pass("Split PMD-mapped pagecache folio to order %d at in-folio offset %d passed\n", order, offset);
633 }
634 }
635
main(int argc,char ** argv)636 int main(int argc, char **argv)
637 {
638 int i;
639 size_t fd_size;
640 char *optional_xfs_path = NULL;
641 char fs_loc_template[] = "/tmp/thp_fs_XXXXXX";
642 const char *fs_loc;
643 bool created_tmp;
644 int offset;
645 unsigned int nr_pages;
646 unsigned int tests;
647
648 ksft_print_header();
649
650 if (geteuid() != 0) {
651 ksft_print_msg("Please run the benchmark as root\n");
652 ksft_finished();
653 }
654
655 if (!thp_is_enabled())
656 ksft_exit_skip("Transparent Hugepages not available\n");
657
658 if (argc > 1)
659 optional_xfs_path = argv[1];
660
661 pagesize = getpagesize();
662 pageshift = ffs(pagesize) - 1;
663 pmd_pagesize = read_pmd_pagesize();
664 if (!pmd_pagesize)
665 ksft_exit_fail_msg("Reading PMD pagesize failed\n");
666
667 nr_pages = pmd_pagesize / pagesize;
668 pmd_order = sz2ord(pmd_pagesize, pagesize);
669
670 expected_orders = (int *)malloc(sizeof(int) * (pmd_order + 1));
671 if (!expected_orders)
672 ksft_exit_fail_msg("Fail to allocate memory: %s\n", strerror(errno));
673
674 tests = 2 + (pmd_order - 1) + (2 * pmd_order) + (pmd_order - 1) * 4 + 2;
675 ksft_set_plan(tests);
676
677 pagemap_fd = open(pagemap_proc, O_RDONLY);
678 if (pagemap_fd == -1)
679 ksft_exit_fail_msg("read pagemap: %s\n", strerror(errno));
680
681 kpageflags_fd = open(kpageflags_proc, O_RDONLY);
682 if (kpageflags_fd == -1)
683 ksft_exit_fail_msg("read kpageflags: %s\n", strerror(errno));
684
685 fd_size = 2 * pmd_pagesize;
686
687 split_pmd_zero_pages();
688
689 for (i = 0; i < pmd_order; i++)
690 if (i != 1)
691 split_pmd_thp_to_order(i);
692
693 split_pte_mapped_thp();
694 for (i = 0; i < pmd_order; i++)
695 split_file_backed_thp(i);
696
697 created_tmp = prepare_thp_fs(optional_xfs_path, fs_loc_template,
698 &fs_loc);
699 for (i = pmd_order - 1; i >= 0; i--)
700 split_thp_in_pagecache_to_order_at(fd_size, fs_loc, i, -1);
701
702 for (i = 0; i < pmd_order; i++)
703 for (offset = 0;
704 offset < nr_pages;
705 offset += MAX(nr_pages / 4, 1 << i))
706 split_thp_in_pagecache_to_order_at(fd_size, fs_loc, i, offset);
707 cleanup_thp_fs(fs_loc, created_tmp);
708
709 close(pagemap_fd);
710 close(kpageflags_fd);
711 free(expected_orders);
712
713 ksft_finished();
714
715 return 0;
716 }
717