1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * A test of splitting PMD THPs and PTE-mapped THPs from a specified virtual 4 * address range in a process via <debugfs>/split_huge_pages interface. 5 */ 6 7 #define _GNU_SOURCE 8 #include <stdio.h> 9 #include <stdlib.h> 10 #include <stdarg.h> 11 #include <unistd.h> 12 #include <inttypes.h> 13 #include <string.h> 14 #include <fcntl.h> 15 #include <sys/mman.h> 16 #include <sys/mount.h> 17 #include <malloc.h> 18 #include <stdbool.h> 19 #include "vm_util.h" 20 #include "../kselftest.h" 21 22 uint64_t pagesize; 23 unsigned int pageshift; 24 uint64_t pmd_pagesize; 25 26 #define SPLIT_DEBUGFS "/sys/kernel/debug/split_huge_pages" 27 #define INPUT_MAX 80 28 29 #define PID_FMT "%d,0x%lx,0x%lx" 30 #define PATH_FMT "%s,0x%lx,0x%lx" 31 32 #define PFN_MASK ((1UL<<55)-1) 33 #define KPF_THP (1UL<<22) 34 35 int is_backed_by_thp(char *vaddr, int pagemap_file, int kpageflags_file) 36 { 37 uint64_t paddr; 38 uint64_t page_flags; 39 40 if (pagemap_file) { 41 pread(pagemap_file, &paddr, sizeof(paddr), 42 ((long)vaddr >> pageshift) * sizeof(paddr)); 43 44 if (kpageflags_file) { 45 pread(kpageflags_file, &page_flags, sizeof(page_flags), 46 (paddr & PFN_MASK) * sizeof(page_flags)); 47 48 return !!(page_flags & KPF_THP); 49 } 50 } 51 return 0; 52 } 53 54 static void write_file(const char *path, const char *buf, size_t buflen) 55 { 56 int fd; 57 ssize_t numwritten; 58 59 fd = open(path, O_WRONLY); 60 if (fd == -1) 61 ksft_exit_fail_msg("%s open failed: %s\n", path, strerror(errno)); 62 63 numwritten = write(fd, buf, buflen - 1); 64 close(fd); 65 if (numwritten < 1) 66 ksft_exit_fail_msg("Write failed\n"); 67 } 68 69 static void write_debugfs(const char *fmt, ...) 70 { 71 char input[INPUT_MAX]; 72 int ret; 73 va_list argp; 74 75 va_start(argp, fmt); 76 ret = vsnprintf(input, INPUT_MAX, fmt, argp); 77 va_end(argp); 78 79 if (ret >= INPUT_MAX) 80 ksft_exit_fail_msg("%s: Debugfs input is too long\n", __func__); 81 82 write_file(SPLIT_DEBUGFS, input, ret + 1); 83 } 84 85 void split_pmd_thp(void) 86 { 87 char *one_page; 88 size_t len = 4 * pmd_pagesize; 89 size_t i; 90 91 one_page = memalign(pmd_pagesize, len); 92 if (!one_page) 93 ksft_exit_fail_msg("Fail to allocate memory: %s\n", strerror(errno)); 94 95 madvise(one_page, len, MADV_HUGEPAGE); 96 97 for (i = 0; i < len; i++) 98 one_page[i] = (char)i; 99 100 if (!check_huge_anon(one_page, 4, pmd_pagesize)) 101 ksft_exit_fail_msg("No THP is allocated\n"); 102 103 /* split all THPs */ 104 write_debugfs(PID_FMT, getpid(), (uint64_t)one_page, 105 (uint64_t)one_page + len); 106 107 for (i = 0; i < len; i++) 108 if (one_page[i] != (char)i) 109 ksft_exit_fail_msg("%ld byte corrupted\n", i); 110 111 112 if (!check_huge_anon(one_page, 0, pmd_pagesize)) 113 ksft_exit_fail_msg("Still AnonHugePages not split\n"); 114 115 ksft_test_result_pass("Split huge pages successful\n"); 116 free(one_page); 117 } 118 119 void split_pte_mapped_thp(void) 120 { 121 char *one_page, *pte_mapped, *pte_mapped2; 122 size_t len = 4 * pmd_pagesize; 123 uint64_t thp_size; 124 size_t i; 125 const char *pagemap_template = "/proc/%d/pagemap"; 126 const char *kpageflags_proc = "/proc/kpageflags"; 127 char pagemap_proc[255]; 128 int pagemap_fd; 129 int kpageflags_fd; 130 131 if (snprintf(pagemap_proc, 255, pagemap_template, getpid()) < 0) 132 ksft_exit_fail_msg("get pagemap proc error: %s\n", strerror(errno)); 133 134 pagemap_fd = open(pagemap_proc, O_RDONLY); 135 if (pagemap_fd == -1) 136 ksft_exit_fail_msg("read pagemap: %s\n", strerror(errno)); 137 138 kpageflags_fd = open(kpageflags_proc, O_RDONLY); 139 if (kpageflags_fd == -1) 140 ksft_exit_fail_msg("read kpageflags: %s\n", strerror(errno)); 141 142 one_page = mmap((void *)(1UL << 30), len, PROT_READ | PROT_WRITE, 143 MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); 144 if (one_page == MAP_FAILED) 145 ksft_exit_fail_msg("Fail to allocate memory: %s\n", strerror(errno)); 146 147 madvise(one_page, len, MADV_HUGEPAGE); 148 149 for (i = 0; i < len; i++) 150 one_page[i] = (char)i; 151 152 if (!check_huge_anon(one_page, 4, pmd_pagesize)) 153 ksft_exit_fail_msg("No THP is allocated\n"); 154 155 /* remap the first pagesize of first THP */ 156 pte_mapped = mremap(one_page, pagesize, pagesize, MREMAP_MAYMOVE); 157 158 /* remap the Nth pagesize of Nth THP */ 159 for (i = 1; i < 4; i++) { 160 pte_mapped2 = mremap(one_page + pmd_pagesize * i + pagesize * i, 161 pagesize, pagesize, 162 MREMAP_MAYMOVE|MREMAP_FIXED, 163 pte_mapped + pagesize * i); 164 if (pte_mapped2 == MAP_FAILED) 165 ksft_exit_fail_msg("mremap failed: %s\n", strerror(errno)); 166 } 167 168 /* smap does not show THPs after mremap, use kpageflags instead */ 169 thp_size = 0; 170 for (i = 0; i < pagesize * 4; i++) 171 if (i % pagesize == 0 && 172 is_backed_by_thp(&pte_mapped[i], pagemap_fd, kpageflags_fd)) 173 thp_size++; 174 175 if (thp_size != 4) 176 ksft_exit_fail_msg("Some THPs are missing during mremap\n"); 177 178 /* split all remapped THPs */ 179 write_debugfs(PID_FMT, getpid(), (uint64_t)pte_mapped, 180 (uint64_t)pte_mapped + pagesize * 4); 181 182 /* smap does not show THPs after mremap, use kpageflags instead */ 183 thp_size = 0; 184 for (i = 0; i < pagesize * 4; i++) { 185 if (pte_mapped[i] != (char)i) 186 ksft_exit_fail_msg("%ld byte corrupted\n", i); 187 188 if (i % pagesize == 0 && 189 is_backed_by_thp(&pte_mapped[i], pagemap_fd, kpageflags_fd)) 190 thp_size++; 191 } 192 193 if (thp_size) 194 ksft_exit_fail_msg("Still %ld THPs not split\n", thp_size); 195 196 ksft_test_result_pass("Split PTE-mapped huge pages successful\n"); 197 munmap(one_page, len); 198 close(pagemap_fd); 199 close(kpageflags_fd); 200 } 201 202 void split_file_backed_thp(void) 203 { 204 int status; 205 int fd; 206 ssize_t num_written; 207 char tmpfs_template[] = "/tmp/thp_split_XXXXXX"; 208 const char *tmpfs_loc = mkdtemp(tmpfs_template); 209 char testfile[INPUT_MAX]; 210 uint64_t pgoff_start = 0, pgoff_end = 1024; 211 212 ksft_print_msg("Please enable pr_debug in split_huge_pages_in_file() for more info.\n"); 213 214 status = mount("tmpfs", tmpfs_loc, "tmpfs", 0, "huge=always,size=4m"); 215 216 if (status) 217 ksft_exit_fail_msg("Unable to create a tmpfs for testing\n"); 218 219 status = snprintf(testfile, INPUT_MAX, "%s/thp_file", tmpfs_loc); 220 if (status >= INPUT_MAX) { 221 ksft_exit_fail_msg("Fail to create file-backed THP split testing file\n"); 222 } 223 224 fd = open(testfile, O_CREAT|O_WRONLY); 225 if (fd == -1) { 226 ksft_perror("Cannot open testing file"); 227 goto cleanup; 228 } 229 230 /* write something to the file, so a file-backed THP can be allocated */ 231 num_written = write(fd, tmpfs_loc, strlen(tmpfs_loc) + 1); 232 close(fd); 233 234 if (num_written < 1) { 235 ksft_perror("Fail to write data to testing file"); 236 goto cleanup; 237 } 238 239 /* split the file-backed THP */ 240 write_debugfs(PATH_FMT, testfile, pgoff_start, pgoff_end); 241 242 status = unlink(testfile); 243 if (status) { 244 ksft_perror("Cannot remove testing file"); 245 goto cleanup; 246 } 247 248 status = umount(tmpfs_loc); 249 if (status) { 250 rmdir(tmpfs_loc); 251 ksft_exit_fail_msg("Unable to umount %s\n", tmpfs_loc); 252 } 253 254 status = rmdir(tmpfs_loc); 255 if (status) 256 ksft_exit_fail_msg("cannot remove tmp dir: %s\n", strerror(errno)); 257 258 ksft_print_msg("Please check dmesg for more information\n"); 259 ksft_test_result_pass("File-backed THP split test done\n"); 260 return; 261 262 cleanup: 263 umount(tmpfs_loc); 264 rmdir(tmpfs_loc); 265 ksft_exit_fail_msg("Error occurred\n"); 266 } 267 268 int main(int argc, char **argv) 269 { 270 ksft_print_header(); 271 272 if (geteuid() != 0) { 273 ksft_print_msg("Please run the benchmark as root\n"); 274 ksft_finished(); 275 } 276 277 ksft_set_plan(3); 278 279 pagesize = getpagesize(); 280 pageshift = ffs(pagesize) - 1; 281 pmd_pagesize = read_pmd_pagesize(); 282 if (!pmd_pagesize) 283 ksft_exit_fail_msg("Reading PMD pagesize failed\n"); 284 285 split_pmd_thp(); 286 split_pte_mapped_thp(); 287 split_file_backed_thp(); 288 289 ksft_finished(); 290 } 291