1 // SPDX-License-Identifier: GPL-2.0 2 #include <string.h> 3 #include <fcntl.h> 4 #include <dirent.h> 5 #include <inttypes.h> 6 #include <sys/ioctl.h> 7 #include <linux/userfaultfd.h> 8 #include <linux/fs.h> 9 #include <sys/syscall.h> 10 #include <unistd.h> 11 #include "../kselftest.h" 12 #include "vm_util.h" 13 14 #define PMD_SIZE_FILE_PATH "/sys/kernel/mm/transparent_hugepage/hpage_pmd_size" 15 #define SMAP_FILE_PATH "/proc/self/smaps" 16 #define STATUS_FILE_PATH "/proc/self/status" 17 #define MAX_LINE_LENGTH 500 18 19 unsigned int __page_size; 20 unsigned int __page_shift; 21 22 uint64_t pagemap_get_entry(int fd, char *start) 23 { 24 const unsigned long pfn = (unsigned long)start / getpagesize(); 25 uint64_t entry; 26 int ret; 27 28 ret = pread(fd, &entry, sizeof(entry), pfn * sizeof(entry)); 29 if (ret != sizeof(entry)) 30 ksft_exit_fail_msg("reading pagemap failed\n"); 31 return entry; 32 } 33 34 static uint64_t __pagemap_scan_get_categories(int fd, char *start, struct page_region *r) 35 { 36 struct pm_scan_arg arg; 37 38 arg.start = (uintptr_t)start; 39 arg.end = (uintptr_t)(start + psize()); 40 arg.vec = (uintptr_t)r; 41 arg.vec_len = 1; 42 arg.flags = 0; 43 arg.size = sizeof(struct pm_scan_arg); 44 arg.max_pages = 0; 45 arg.category_inverted = 0; 46 arg.category_mask = 0; 47 arg.category_anyof_mask = PAGE_IS_WPALLOWED | PAGE_IS_WRITTEN | PAGE_IS_FILE | 48 PAGE_IS_PRESENT | PAGE_IS_SWAPPED | PAGE_IS_PFNZERO | 49 PAGE_IS_HUGE | PAGE_IS_SOFT_DIRTY; 50 arg.return_mask = arg.category_anyof_mask; 51 52 return ioctl(fd, PAGEMAP_SCAN, &arg); 53 } 54 55 static uint64_t pagemap_scan_get_categories(int fd, char *start) 56 { 57 struct page_region r; 58 long ret; 59 60 ret = __pagemap_scan_get_categories(fd, start, &r); 61 if (ret < 0) 62 ksft_exit_fail_msg("PAGEMAP_SCAN failed: %s\n", strerror(errno)); 63 if (ret == 0) 64 return 0; 65 return r.categories; 66 } 67 68 /* `start` is any valid address. */ 69 static bool pagemap_scan_supported(int fd, char *start) 70 { 71 static int supported = -1; 72 int ret; 73 74 if (supported != -1) 75 return supported; 76 77 /* Provide an invalid address in order to trigger EFAULT. */ 78 ret = __pagemap_scan_get_categories(fd, start, (struct page_region *) ~0UL); 79 if (ret == 0) 80 ksft_exit_fail_msg("PAGEMAP_SCAN succeeded unexpectedly\n"); 81 82 supported = errno == EFAULT; 83 84 return supported; 85 } 86 87 static bool page_entry_is(int fd, char *start, char *desc, 88 uint64_t pagemap_flags, uint64_t pagescan_flags) 89 { 90 bool m = pagemap_get_entry(fd, start) & pagemap_flags; 91 92 if (pagemap_scan_supported(fd, start)) { 93 bool s = pagemap_scan_get_categories(fd, start) & pagescan_flags; 94 95 if (m == s) 96 return m; 97 98 ksft_exit_fail_msg( 99 "read and ioctl return unmatched results for %s: %d %d", desc, m, s); 100 } 101 return m; 102 } 103 104 bool pagemap_is_softdirty(int fd, char *start) 105 { 106 return page_entry_is(fd, start, "soft-dirty", 107 PM_SOFT_DIRTY, PAGE_IS_SOFT_DIRTY); 108 } 109 110 bool pagemap_is_swapped(int fd, char *start) 111 { 112 return page_entry_is(fd, start, "swap", PM_SWAP, PAGE_IS_SWAPPED); 113 } 114 115 bool pagemap_is_populated(int fd, char *start) 116 { 117 return page_entry_is(fd, start, "populated", 118 PM_PRESENT | PM_SWAP, 119 PAGE_IS_PRESENT | PAGE_IS_SWAPPED); 120 } 121 122 unsigned long pagemap_get_pfn(int fd, char *start) 123 { 124 uint64_t entry = pagemap_get_entry(fd, start); 125 126 /* If present (63th bit), PFN is at bit 0 -- 54. */ 127 if (entry & PM_PRESENT) 128 return entry & 0x007fffffffffffffull; 129 return -1ul; 130 } 131 132 void clear_softdirty(void) 133 { 134 int ret; 135 const char *ctrl = "4"; 136 int fd = open("/proc/self/clear_refs", O_WRONLY); 137 138 if (fd < 0) 139 ksft_exit_fail_msg("opening clear_refs failed\n"); 140 ret = write(fd, ctrl, strlen(ctrl)); 141 close(fd); 142 if (ret != (signed int)strlen(ctrl)) 143 ksft_exit_fail_msg("writing clear_refs failed\n"); 144 } 145 146 bool check_for_pattern(FILE *fp, const char *pattern, char *buf, size_t len) 147 { 148 while (fgets(buf, len, fp)) { 149 if (!strncmp(buf, pattern, strlen(pattern))) 150 return true; 151 } 152 return false; 153 } 154 155 uint64_t read_pmd_pagesize(void) 156 { 157 int fd; 158 char buf[20]; 159 ssize_t num_read; 160 161 fd = open(PMD_SIZE_FILE_PATH, O_RDONLY); 162 if (fd == -1) 163 return 0; 164 165 num_read = read(fd, buf, 19); 166 if (num_read < 1) { 167 close(fd); 168 return 0; 169 } 170 buf[num_read] = '\0'; 171 close(fd); 172 173 return strtoul(buf, NULL, 10); 174 } 175 176 unsigned long rss_anon(void) 177 { 178 unsigned long rss_anon = 0; 179 FILE *fp; 180 char buffer[MAX_LINE_LENGTH]; 181 182 fp = fopen(STATUS_FILE_PATH, "r"); 183 if (!fp) 184 ksft_exit_fail_msg("%s: Failed to open file %s\n", __func__, STATUS_FILE_PATH); 185 186 if (!check_for_pattern(fp, "RssAnon:", buffer, sizeof(buffer))) 187 goto err_out; 188 189 if (sscanf(buffer, "RssAnon:%10lu kB", &rss_anon) != 1) 190 ksft_exit_fail_msg("Reading status error\n"); 191 192 err_out: 193 fclose(fp); 194 return rss_anon; 195 } 196 197 char *__get_smap_entry(void *addr, const char *pattern, char *buf, size_t len) 198 { 199 int ret; 200 FILE *fp; 201 char *entry = NULL; 202 char addr_pattern[MAX_LINE_LENGTH]; 203 204 ret = snprintf(addr_pattern, MAX_LINE_LENGTH, "%08lx-", 205 (unsigned long) addr); 206 if (ret >= MAX_LINE_LENGTH) 207 ksft_exit_fail_msg("%s: Pattern is too long\n", __func__); 208 209 fp = fopen(SMAP_FILE_PATH, "r"); 210 if (!fp) 211 ksft_exit_fail_msg("%s: Failed to open file %s\n", __func__, SMAP_FILE_PATH); 212 213 if (!check_for_pattern(fp, addr_pattern, buf, len)) 214 goto err_out; 215 216 /* Fetch the pattern in the same block */ 217 if (!check_for_pattern(fp, pattern, buf, len)) 218 goto err_out; 219 220 /* Trim trailing newline */ 221 entry = strchr(buf, '\n'); 222 if (entry) 223 *entry = '\0'; 224 225 entry = buf + strlen(pattern); 226 227 err_out: 228 fclose(fp); 229 return entry; 230 } 231 232 bool __check_huge(void *addr, char *pattern, int nr_hpages, 233 uint64_t hpage_size) 234 { 235 char buffer[MAX_LINE_LENGTH]; 236 uint64_t thp = -1; 237 char *entry; 238 239 entry = __get_smap_entry(addr, pattern, buffer, sizeof(buffer)); 240 if (!entry) 241 goto err_out; 242 243 if (sscanf(entry, "%9" SCNu64 " kB", &thp) != 1) 244 ksft_exit_fail_msg("Reading smap error\n"); 245 246 err_out: 247 return thp == (nr_hpages * (hpage_size >> 10)); 248 } 249 250 bool check_huge_anon(void *addr, int nr_hpages, uint64_t hpage_size) 251 { 252 return __check_huge(addr, "AnonHugePages: ", nr_hpages, hpage_size); 253 } 254 255 bool check_huge_file(void *addr, int nr_hpages, uint64_t hpage_size) 256 { 257 return __check_huge(addr, "FilePmdMapped:", nr_hpages, hpage_size); 258 } 259 260 bool check_huge_shmem(void *addr, int nr_hpages, uint64_t hpage_size) 261 { 262 return __check_huge(addr, "ShmemPmdMapped:", nr_hpages, hpage_size); 263 } 264 265 int64_t allocate_transhuge(void *ptr, int pagemap_fd) 266 { 267 uint64_t ent[2]; 268 269 /* drop pmd */ 270 if (mmap(ptr, HPAGE_SIZE, PROT_READ | PROT_WRITE, 271 MAP_FIXED | MAP_ANONYMOUS | 272 MAP_NORESERVE | MAP_PRIVATE, -1, 0) != ptr) 273 ksft_exit_fail_msg("mmap transhuge\n"); 274 275 if (madvise(ptr, HPAGE_SIZE, MADV_HUGEPAGE)) 276 ksft_exit_fail_msg("MADV_HUGEPAGE\n"); 277 278 /* allocate transparent huge page */ 279 *(volatile void **)ptr = ptr; 280 281 if (pread(pagemap_fd, ent, sizeof(ent), 282 (uintptr_t)ptr >> (pshift() - 3)) != sizeof(ent)) 283 ksft_exit_fail_msg("read pagemap\n"); 284 285 if (PAGEMAP_PRESENT(ent[0]) && PAGEMAP_PRESENT(ent[1]) && 286 PAGEMAP_PFN(ent[0]) + 1 == PAGEMAP_PFN(ent[1]) && 287 !(PAGEMAP_PFN(ent[0]) & ((1 << (HPAGE_SHIFT - pshift())) - 1))) 288 return PAGEMAP_PFN(ent[0]); 289 290 return -1; 291 } 292 293 unsigned long default_huge_page_size(void) 294 { 295 unsigned long hps = 0; 296 char *line = NULL; 297 size_t linelen = 0; 298 FILE *f = fopen("/proc/meminfo", "r"); 299 300 if (!f) 301 return 0; 302 while (getline(&line, &linelen, f) > 0) { 303 if (sscanf(line, "Hugepagesize: %lu kB", &hps) == 1) { 304 hps <<= 10; 305 break; 306 } 307 } 308 309 free(line); 310 fclose(f); 311 return hps; 312 } 313 314 int detect_hugetlb_page_sizes(size_t sizes[], int max) 315 { 316 DIR *dir = opendir("/sys/kernel/mm/hugepages/"); 317 int count = 0; 318 319 if (!dir) 320 return 0; 321 322 while (count < max) { 323 struct dirent *entry = readdir(dir); 324 size_t kb; 325 326 if (!entry) 327 break; 328 if (entry->d_type != DT_DIR) 329 continue; 330 if (sscanf(entry->d_name, "hugepages-%zukB", &kb) != 1) 331 continue; 332 sizes[count++] = kb * 1024; 333 ksft_print_msg("[INFO] detected hugetlb page size: %zu KiB\n", 334 kb); 335 } 336 closedir(dir); 337 return count; 338 } 339 340 /* If `ioctls' non-NULL, the allowed ioctls will be returned into the var */ 341 int uffd_register_with_ioctls(int uffd, void *addr, uint64_t len, 342 bool miss, bool wp, bool minor, uint64_t *ioctls) 343 { 344 struct uffdio_register uffdio_register = { 0 }; 345 uint64_t mode = 0; 346 int ret = 0; 347 348 if (miss) 349 mode |= UFFDIO_REGISTER_MODE_MISSING; 350 if (wp) 351 mode |= UFFDIO_REGISTER_MODE_WP; 352 if (minor) 353 mode |= UFFDIO_REGISTER_MODE_MINOR; 354 355 uffdio_register.range.start = (unsigned long)addr; 356 uffdio_register.range.len = len; 357 uffdio_register.mode = mode; 358 359 if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register) == -1) 360 ret = -errno; 361 else if (ioctls) 362 *ioctls = uffdio_register.ioctls; 363 364 return ret; 365 } 366 367 int uffd_register(int uffd, void *addr, uint64_t len, 368 bool miss, bool wp, bool minor) 369 { 370 return uffd_register_with_ioctls(uffd, addr, len, 371 miss, wp, minor, NULL); 372 } 373 374 int uffd_unregister(int uffd, void *addr, uint64_t len) 375 { 376 struct uffdio_range range = { .start = (uintptr_t)addr, .len = len }; 377 int ret = 0; 378 379 if (ioctl(uffd, UFFDIO_UNREGISTER, &range) == -1) 380 ret = -errno; 381 382 return ret; 383 } 384 385 unsigned long get_free_hugepages(void) 386 { 387 unsigned long fhp = 0; 388 char *line = NULL; 389 size_t linelen = 0; 390 FILE *f = fopen("/proc/meminfo", "r"); 391 392 if (!f) 393 return fhp; 394 while (getline(&line, &linelen, f) > 0) { 395 if (sscanf(line, "HugePages_Free: %lu", &fhp) == 1) 396 break; 397 } 398 399 free(line); 400 fclose(f); 401 return fhp; 402 } 403 404 bool check_vmflag_io(void *addr) 405 { 406 char buffer[MAX_LINE_LENGTH]; 407 const char *flags; 408 size_t flaglen; 409 410 flags = __get_smap_entry(addr, "VmFlags:", buffer, sizeof(buffer)); 411 if (!flags) 412 ksft_exit_fail_msg("%s: No VmFlags for %p\n", __func__, addr); 413 414 while (true) { 415 flags += strspn(flags, " "); 416 417 flaglen = strcspn(flags, " "); 418 if (!flaglen) 419 return false; 420 421 if (flaglen == strlen("io") && !memcmp(flags, "io", flaglen)) 422 return true; 423 424 flags += flaglen; 425 } 426 } 427