1 // SPDX-License-Identifier: GPL-2.0 2 #include <string.h> 3 #include <errno.h> 4 #include <fcntl.h> 5 #include <dirent.h> 6 #include <inttypes.h> 7 #include <sys/ioctl.h> 8 #include <linux/userfaultfd.h> 9 #include <linux/fs.h> 10 #include <sys/syscall.h> 11 #include <unistd.h> 12 #include "../kselftest.h" 13 #include "vm_util.h" 14 15 #define PMD_SIZE_FILE_PATH "/sys/kernel/mm/transparent_hugepage/hpage_pmd_size" 16 #define SMAP_FILE_PATH "/proc/self/smaps" 17 #define STATUS_FILE_PATH "/proc/self/status" 18 #define MAX_LINE_LENGTH 500 19 20 unsigned int __page_size; 21 unsigned int __page_shift; 22 23 uint64_t pagemap_get_entry(int fd, char *start) 24 { 25 const unsigned long pfn = (unsigned long)start / getpagesize(); 26 uint64_t entry; 27 int ret; 28 29 ret = pread(fd, &entry, sizeof(entry), pfn * sizeof(entry)); 30 if (ret != sizeof(entry)) 31 ksft_exit_fail_msg("reading pagemap failed\n"); 32 return entry; 33 } 34 35 static uint64_t __pagemap_scan_get_categories(int fd, char *start, struct page_region *r) 36 { 37 struct pm_scan_arg arg; 38 39 arg.start = (uintptr_t)start; 40 arg.end = (uintptr_t)(start + psize()); 41 arg.vec = (uintptr_t)r; 42 arg.vec_len = 1; 43 arg.flags = 0; 44 arg.size = sizeof(struct pm_scan_arg); 45 arg.max_pages = 0; 46 arg.category_inverted = 0; 47 arg.category_mask = 0; 48 arg.category_anyof_mask = PAGE_IS_WPALLOWED | PAGE_IS_WRITTEN | PAGE_IS_FILE | 49 PAGE_IS_PRESENT | PAGE_IS_SWAPPED | PAGE_IS_PFNZERO | 50 PAGE_IS_HUGE | PAGE_IS_SOFT_DIRTY; 51 arg.return_mask = arg.category_anyof_mask; 52 53 return ioctl(fd, PAGEMAP_SCAN, &arg); 54 } 55 56 static uint64_t pagemap_scan_get_categories(int fd, char *start) 57 { 58 struct page_region r; 59 long ret; 60 61 ret = __pagemap_scan_get_categories(fd, start, &r); 62 if (ret < 0) 63 ksft_exit_fail_msg("PAGEMAP_SCAN failed: %s\n", strerror(errno)); 64 if (ret == 0) 65 return 0; 66 return r.categories; 67 } 68 69 /* `start` is any valid address. */ 70 static bool pagemap_scan_supported(int fd, char *start) 71 { 72 static int supported = -1; 73 int ret; 74 75 if (supported != -1) 76 return supported; 77 78 /* Provide an invalid address in order to trigger EFAULT. */ 79 ret = __pagemap_scan_get_categories(fd, start, (struct page_region *) ~0UL); 80 if (ret == 0) 81 ksft_exit_fail_msg("PAGEMAP_SCAN succeeded unexpectedly\n"); 82 83 supported = errno == EFAULT; 84 85 return supported; 86 } 87 88 static bool page_entry_is(int fd, char *start, char *desc, 89 uint64_t pagemap_flags, uint64_t pagescan_flags) 90 { 91 bool m = pagemap_get_entry(fd, start) & pagemap_flags; 92 93 if (pagemap_scan_supported(fd, start)) { 94 bool s = pagemap_scan_get_categories(fd, start) & pagescan_flags; 95 96 if (m == s) 97 return m; 98 99 ksft_exit_fail_msg( 100 "read and ioctl return unmatched results for %s: %d %d", desc, m, s); 101 } 102 return m; 103 } 104 105 bool pagemap_is_softdirty(int fd, char *start) 106 { 107 return page_entry_is(fd, start, "soft-dirty", 108 PM_SOFT_DIRTY, PAGE_IS_SOFT_DIRTY); 109 } 110 111 bool pagemap_is_swapped(int fd, char *start) 112 { 113 return page_entry_is(fd, start, "swap", PM_SWAP, PAGE_IS_SWAPPED); 114 } 115 116 bool pagemap_is_populated(int fd, char *start) 117 { 118 return page_entry_is(fd, start, "populated", 119 PM_PRESENT | PM_SWAP, 120 PAGE_IS_PRESENT | PAGE_IS_SWAPPED); 121 } 122 123 unsigned long pagemap_get_pfn(int fd, char *start) 124 { 125 uint64_t entry = pagemap_get_entry(fd, start); 126 127 /* If present (63th bit), PFN is at bit 0 -- 54. */ 128 if (entry & PM_PRESENT) 129 return entry & 0x007fffffffffffffull; 130 return -1ul; 131 } 132 133 void clear_softdirty(void) 134 { 135 int ret; 136 const char *ctrl = "4"; 137 int fd = open("/proc/self/clear_refs", O_WRONLY); 138 139 if (fd < 0) 140 ksft_exit_fail_msg("opening clear_refs failed\n"); 141 ret = write(fd, ctrl, strlen(ctrl)); 142 close(fd); 143 if (ret != (signed int)strlen(ctrl)) 144 ksft_exit_fail_msg("writing clear_refs failed\n"); 145 } 146 147 bool check_for_pattern(FILE *fp, const char *pattern, char *buf, size_t len) 148 { 149 while (fgets(buf, len, fp)) { 150 if (!strncmp(buf, pattern, strlen(pattern))) 151 return true; 152 } 153 return false; 154 } 155 156 uint64_t read_pmd_pagesize(void) 157 { 158 int fd; 159 char buf[20]; 160 ssize_t num_read; 161 162 fd = open(PMD_SIZE_FILE_PATH, O_RDONLY); 163 if (fd == -1) 164 return 0; 165 166 num_read = read(fd, buf, 19); 167 if (num_read < 1) { 168 close(fd); 169 return 0; 170 } 171 buf[num_read] = '\0'; 172 close(fd); 173 174 return strtoul(buf, NULL, 10); 175 } 176 177 unsigned long rss_anon(void) 178 { 179 unsigned long rss_anon = 0; 180 FILE *fp; 181 char buffer[MAX_LINE_LENGTH]; 182 183 fp = fopen(STATUS_FILE_PATH, "r"); 184 if (!fp) 185 ksft_exit_fail_msg("%s: Failed to open file %s\n", __func__, STATUS_FILE_PATH); 186 187 if (!check_for_pattern(fp, "RssAnon:", buffer, sizeof(buffer))) 188 goto err_out; 189 190 if (sscanf(buffer, "RssAnon:%10lu kB", &rss_anon) != 1) 191 ksft_exit_fail_msg("Reading status error\n"); 192 193 err_out: 194 fclose(fp); 195 return rss_anon; 196 } 197 198 char *__get_smap_entry(void *addr, const char *pattern, char *buf, size_t len) 199 { 200 int ret; 201 FILE *fp; 202 char *entry = NULL; 203 char addr_pattern[MAX_LINE_LENGTH]; 204 205 ret = snprintf(addr_pattern, MAX_LINE_LENGTH, "%08lx-", 206 (unsigned long) addr); 207 if (ret >= MAX_LINE_LENGTH) 208 ksft_exit_fail_msg("%s: Pattern is too long\n", __func__); 209 210 fp = fopen(SMAP_FILE_PATH, "r"); 211 if (!fp) 212 ksft_exit_fail_msg("%s: Failed to open file %s\n", __func__, SMAP_FILE_PATH); 213 214 if (!check_for_pattern(fp, addr_pattern, buf, len)) 215 goto err_out; 216 217 /* Fetch the pattern in the same block */ 218 if (!check_for_pattern(fp, pattern, buf, len)) 219 goto err_out; 220 221 /* Trim trailing newline */ 222 entry = strchr(buf, '\n'); 223 if (entry) 224 *entry = '\0'; 225 226 entry = buf + strlen(pattern); 227 228 err_out: 229 fclose(fp); 230 return entry; 231 } 232 233 bool __check_huge(void *addr, char *pattern, int nr_hpages, 234 uint64_t hpage_size) 235 { 236 char buffer[MAX_LINE_LENGTH]; 237 uint64_t thp = -1; 238 char *entry; 239 240 entry = __get_smap_entry(addr, pattern, buffer, sizeof(buffer)); 241 if (!entry) 242 goto err_out; 243 244 if (sscanf(entry, "%9" SCNu64 " kB", &thp) != 1) 245 ksft_exit_fail_msg("Reading smap error\n"); 246 247 err_out: 248 return thp == (nr_hpages * (hpage_size >> 10)); 249 } 250 251 bool check_huge_anon(void *addr, int nr_hpages, uint64_t hpage_size) 252 { 253 return __check_huge(addr, "AnonHugePages: ", nr_hpages, hpage_size); 254 } 255 256 bool check_huge_file(void *addr, int nr_hpages, uint64_t hpage_size) 257 { 258 return __check_huge(addr, "FilePmdMapped:", nr_hpages, hpage_size); 259 } 260 261 bool check_huge_shmem(void *addr, int nr_hpages, uint64_t hpage_size) 262 { 263 return __check_huge(addr, "ShmemPmdMapped:", nr_hpages, hpage_size); 264 } 265 266 int64_t allocate_transhuge(void *ptr, int pagemap_fd) 267 { 268 uint64_t ent[2]; 269 270 /* drop pmd */ 271 if (mmap(ptr, HPAGE_SIZE, PROT_READ | PROT_WRITE, 272 MAP_FIXED | MAP_ANONYMOUS | 273 MAP_NORESERVE | MAP_PRIVATE, -1, 0) != ptr) 274 ksft_exit_fail_msg("mmap transhuge\n"); 275 276 if (madvise(ptr, HPAGE_SIZE, MADV_HUGEPAGE)) 277 ksft_exit_fail_msg("MADV_HUGEPAGE\n"); 278 279 /* allocate transparent huge page */ 280 *(volatile void **)ptr = ptr; 281 282 if (pread(pagemap_fd, ent, sizeof(ent), 283 (uintptr_t)ptr >> (pshift() - 3)) != sizeof(ent)) 284 ksft_exit_fail_msg("read pagemap\n"); 285 286 if (PAGEMAP_PRESENT(ent[0]) && PAGEMAP_PRESENT(ent[1]) && 287 PAGEMAP_PFN(ent[0]) + 1 == PAGEMAP_PFN(ent[1]) && 288 !(PAGEMAP_PFN(ent[0]) & ((1 << (HPAGE_SHIFT - pshift())) - 1))) 289 return PAGEMAP_PFN(ent[0]); 290 291 return -1; 292 } 293 294 unsigned long default_huge_page_size(void) 295 { 296 unsigned long hps = 0; 297 char *line = NULL; 298 size_t linelen = 0; 299 FILE *f = fopen("/proc/meminfo", "r"); 300 301 if (!f) 302 return 0; 303 while (getline(&line, &linelen, f) > 0) { 304 if (sscanf(line, "Hugepagesize: %lu kB", &hps) == 1) { 305 hps <<= 10; 306 break; 307 } 308 } 309 310 free(line); 311 fclose(f); 312 return hps; 313 } 314 315 int detect_hugetlb_page_sizes(size_t sizes[], int max) 316 { 317 DIR *dir = opendir("/sys/kernel/mm/hugepages/"); 318 int count = 0; 319 320 if (!dir) 321 return 0; 322 323 while (count < max) { 324 struct dirent *entry = readdir(dir); 325 size_t kb; 326 327 if (!entry) 328 break; 329 if (entry->d_type != DT_DIR) 330 continue; 331 if (sscanf(entry->d_name, "hugepages-%zukB", &kb) != 1) 332 continue; 333 sizes[count++] = kb * 1024; 334 ksft_print_msg("[INFO] detected hugetlb page size: %zu KiB\n", 335 kb); 336 } 337 closedir(dir); 338 return count; 339 } 340 341 int pageflags_get(unsigned long pfn, int kpageflags_fd, uint64_t *flags) 342 { 343 size_t count; 344 345 count = pread(kpageflags_fd, flags, sizeof(*flags), 346 pfn * sizeof(*flags)); 347 348 if (count != sizeof(*flags)) 349 return -1; 350 351 return 0; 352 } 353 354 /* If `ioctls' non-NULL, the allowed ioctls will be returned into the var */ 355 int uffd_register_with_ioctls(int uffd, void *addr, uint64_t len, 356 bool miss, bool wp, bool minor, uint64_t *ioctls) 357 { 358 struct uffdio_register uffdio_register = { 0 }; 359 uint64_t mode = 0; 360 int ret = 0; 361 362 if (miss) 363 mode |= UFFDIO_REGISTER_MODE_MISSING; 364 if (wp) 365 mode |= UFFDIO_REGISTER_MODE_WP; 366 if (minor) 367 mode |= UFFDIO_REGISTER_MODE_MINOR; 368 369 uffdio_register.range.start = (unsigned long)addr; 370 uffdio_register.range.len = len; 371 uffdio_register.mode = mode; 372 373 if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register) == -1) 374 ret = -errno; 375 else if (ioctls) 376 *ioctls = uffdio_register.ioctls; 377 378 return ret; 379 } 380 381 int uffd_register(int uffd, void *addr, uint64_t len, 382 bool miss, bool wp, bool minor) 383 { 384 return uffd_register_with_ioctls(uffd, addr, len, 385 miss, wp, minor, NULL); 386 } 387 388 int uffd_unregister(int uffd, void *addr, uint64_t len) 389 { 390 struct uffdio_range range = { .start = (uintptr_t)addr, .len = len }; 391 int ret = 0; 392 393 if (ioctl(uffd, UFFDIO_UNREGISTER, &range) == -1) 394 ret = -errno; 395 396 return ret; 397 } 398 399 unsigned long get_free_hugepages(void) 400 { 401 unsigned long fhp = 0; 402 char *line = NULL; 403 size_t linelen = 0; 404 FILE *f = fopen("/proc/meminfo", "r"); 405 406 if (!f) 407 return fhp; 408 while (getline(&line, &linelen, f) > 0) { 409 if (sscanf(line, "HugePages_Free: %lu", &fhp) == 1) 410 break; 411 } 412 413 free(line); 414 fclose(f); 415 return fhp; 416 } 417 418 static bool check_vmflag(void *addr, const char *flag) 419 { 420 char buffer[MAX_LINE_LENGTH]; 421 const char *flags; 422 size_t flaglen; 423 424 flags = __get_smap_entry(addr, "VmFlags:", buffer, sizeof(buffer)); 425 if (!flags) 426 ksft_exit_fail_msg("%s: No VmFlags for %p\n", __func__, addr); 427 428 while (true) { 429 flags += strspn(flags, " "); 430 431 flaglen = strcspn(flags, " "); 432 if (!flaglen) 433 return false; 434 435 if (flaglen == strlen(flag) && !memcmp(flags, flag, flaglen)) 436 return true; 437 438 flags += flaglen; 439 } 440 } 441 442 bool check_vmflag_io(void *addr) 443 { 444 return check_vmflag(addr, "io"); 445 } 446 447 bool check_vmflag_pfnmap(void *addr) 448 { 449 return check_vmflag(addr, "pf"); 450 } 451 452 /* 453 * Open an fd at /proc/$pid/maps and configure procmap_out ready for 454 * PROCMAP_QUERY query. Returns 0 on success, or an error code otherwise. 455 */ 456 int open_procmap(pid_t pid, struct procmap_fd *procmap_out) 457 { 458 char path[256]; 459 int ret = 0; 460 461 memset(procmap_out, '\0', sizeof(*procmap_out)); 462 sprintf(path, "/proc/%d/maps", pid); 463 procmap_out->query.size = sizeof(procmap_out->query); 464 procmap_out->fd = open(path, O_RDONLY); 465 if (procmap_out->fd < 0) 466 ret = -errno; 467 468 return ret; 469 } 470 471 /* Perform PROCMAP_QUERY. Returns 0 on success, or an error code otherwise. */ 472 int query_procmap(struct procmap_fd *procmap) 473 { 474 int ret = 0; 475 476 if (ioctl(procmap->fd, PROCMAP_QUERY, &procmap->query) == -1) 477 ret = -errno; 478 479 return ret; 480 } 481 482 /* 483 * Try to find the VMA at specified address, returns true if found, false if not 484 * found, and the test is failed if any other error occurs. 485 * 486 * On success, procmap->query is populated with the results. 487 */ 488 bool find_vma_procmap(struct procmap_fd *procmap, void *address) 489 { 490 int err; 491 492 procmap->query.query_flags = 0; 493 procmap->query.query_addr = (unsigned long)address; 494 err = query_procmap(procmap); 495 if (!err) 496 return true; 497 498 if (err != -ENOENT) 499 ksft_exit_fail_msg("%s: Error %d on ioctl(PROCMAP_QUERY)\n", 500 __func__, err); 501 return false; 502 } 503 504 /* 505 * Close fd used by PROCMAP_QUERY mechanism. Returns 0 on success, or an error 506 * code otherwise. 507 */ 508 int close_procmap(struct procmap_fd *procmap) 509 { 510 return close(procmap->fd); 511 } 512 513 int write_sysfs(const char *file_path, unsigned long val) 514 { 515 FILE *f = fopen(file_path, "w"); 516 517 if (!f) { 518 fprintf(stderr, "f %s\n", file_path); 519 perror("fopen"); 520 return 1; 521 } 522 if (fprintf(f, "%lu", val) < 0) { 523 perror("fprintf"); 524 fclose(f); 525 return 1; 526 } 527 fclose(f); 528 529 return 0; 530 } 531 532 int read_sysfs(const char *file_path, unsigned long *val) 533 { 534 FILE *f = fopen(file_path, "r"); 535 536 if (!f) { 537 fprintf(stderr, "f %s\n", file_path); 538 perror("fopen"); 539 return 1; 540 } 541 if (fscanf(f, "%lu", val) != 1) { 542 perror("fscanf"); 543 fclose(f); 544 return 1; 545 } 546 fclose(f); 547 548 return 0; 549 } 550 551 void *sys_mremap(void *old_address, unsigned long old_size, 552 unsigned long new_size, int flags, void *new_address) 553 { 554 return (void *)syscall(__NR_mremap, (unsigned long)old_address, 555 old_size, new_size, flags, 556 (unsigned long)new_address); 557 } 558 559 bool detect_huge_zeropage(void) 560 { 561 int fd = open("/sys/kernel/mm/transparent_hugepage/use_zero_page", 562 O_RDONLY); 563 bool enabled = 0; 564 char buf[15]; 565 int ret; 566 567 if (fd < 0) 568 return 0; 569 570 ret = pread(fd, buf, sizeof(buf), 0); 571 if (ret > 0 && ret < sizeof(buf)) { 572 buf[ret] = 0; 573 574 if (strtoul(buf, NULL, 10) == 1) 575 enabled = 1; 576 } 577 578 close(fd); 579 return enabled; 580 } 581 582 long ksm_get_self_zero_pages(void) 583 { 584 int proc_self_ksm_stat_fd; 585 char buf[200]; 586 char *substr_ksm_zero; 587 size_t value_pos; 588 ssize_t read_size; 589 590 proc_self_ksm_stat_fd = open("/proc/self/ksm_stat", O_RDONLY); 591 if (proc_self_ksm_stat_fd < 0) 592 return -errno; 593 594 read_size = pread(proc_self_ksm_stat_fd, buf, sizeof(buf) - 1, 0); 595 close(proc_self_ksm_stat_fd); 596 if (read_size < 0) 597 return -errno; 598 599 buf[read_size] = 0; 600 601 substr_ksm_zero = strstr(buf, "ksm_zero_pages"); 602 if (!substr_ksm_zero) 603 return 0; 604 605 value_pos = strcspn(substr_ksm_zero, "0123456789"); 606 return strtol(substr_ksm_zero + value_pos, NULL, 10); 607 } 608 609 long ksm_get_self_merging_pages(void) 610 { 611 int proc_self_ksm_merging_pages_fd; 612 char buf[10]; 613 ssize_t ret; 614 615 proc_self_ksm_merging_pages_fd = open("/proc/self/ksm_merging_pages", 616 O_RDONLY); 617 if (proc_self_ksm_merging_pages_fd < 0) 618 return -errno; 619 620 ret = pread(proc_self_ksm_merging_pages_fd, buf, sizeof(buf) - 1, 0); 621 close(proc_self_ksm_merging_pages_fd); 622 if (ret <= 0) 623 return -errno; 624 buf[ret] = 0; 625 626 return strtol(buf, NULL, 10); 627 } 628 629 long ksm_get_full_scans(void) 630 { 631 int ksm_full_scans_fd; 632 char buf[10]; 633 ssize_t ret; 634 635 ksm_full_scans_fd = open("/sys/kernel/mm/ksm/full_scans", O_RDONLY); 636 if (ksm_full_scans_fd < 0) 637 return -errno; 638 639 ret = pread(ksm_full_scans_fd, buf, sizeof(buf) - 1, 0); 640 close(ksm_full_scans_fd); 641 if (ret <= 0) 642 return -errno; 643 buf[ret] = 0; 644 645 return strtol(buf, NULL, 10); 646 } 647 648 int ksm_use_zero_pages(void) 649 { 650 int ksm_use_zero_pages_fd; 651 ssize_t ret; 652 653 ksm_use_zero_pages_fd = open("/sys/kernel/mm/ksm/use_zero_pages", O_RDWR); 654 if (ksm_use_zero_pages_fd < 0) 655 return -errno; 656 657 ret = write(ksm_use_zero_pages_fd, "1", 1); 658 close(ksm_use_zero_pages_fd); 659 return ret == 1 ? 0 : -errno; 660 } 661 662 int ksm_start(void) 663 { 664 int ksm_fd; 665 ssize_t ret; 666 long start_scans, end_scans; 667 668 ksm_fd = open("/sys/kernel/mm/ksm/run", O_RDWR); 669 if (ksm_fd < 0) 670 return -errno; 671 672 /* Wait for two full scans such that any possible merging happened. */ 673 start_scans = ksm_get_full_scans(); 674 if (start_scans < 0) { 675 close(ksm_fd); 676 return start_scans; 677 } 678 ret = write(ksm_fd, "1", 1); 679 close(ksm_fd); 680 if (ret != 1) 681 return -errno; 682 do { 683 end_scans = ksm_get_full_scans(); 684 if (end_scans < 0) 685 return end_scans; 686 } while (end_scans < start_scans + 2); 687 688 return 0; 689 } 690 691 int ksm_stop(void) 692 { 693 int ksm_fd; 694 ssize_t ret; 695 696 ksm_fd = open("/sys/kernel/mm/ksm/run", O_RDWR); 697 if (ksm_fd < 0) 698 return -errno; 699 700 ret = write(ksm_fd, "2", 1); 701 close(ksm_fd); 702 return ret == 1 ? 0 : -errno; 703 } 704