1 // SPDX-License-Identifier: GPL-2.0 2 #include <string.h> 3 #include <errno.h> 4 #include <fcntl.h> 5 #include <inttypes.h> 6 #include <sys/ioctl.h> 7 #include <linux/userfaultfd.h> 8 #include <linux/fs.h> 9 #include <sys/syscall.h> 10 #include <unistd.h> 11 #include "kselftest.h" 12 #include "vm_util.h" 13 14 #define PMD_SIZE_FILE_PATH "/sys/kernel/mm/transparent_hugepage/hpage_pmd_size" 15 #define SMAP_FILE_PATH "/proc/self/smaps" 16 #define STATUS_FILE_PATH "/proc/self/status" 17 #define MAX_LINE_LENGTH 500 18 #define PAGEMAP_PATH "/proc/self/pagemap" 19 #define KPAGEFLAGS_PATH "/proc/kpageflags" 20 #define MAX_NR_ORDERS 20 21 22 unsigned int __page_size; 23 unsigned int __page_shift; 24 25 uint64_t pagemap_get_entry(int fd, char *start) 26 { 27 const unsigned long pfn = (unsigned long)start / getpagesize(); 28 uint64_t entry; 29 int ret; 30 31 ret = pread(fd, &entry, sizeof(entry), pfn * sizeof(entry)); 32 if (ret != sizeof(entry)) 33 ksft_exit_fail_msg("reading pagemap failed\n"); 34 return entry; 35 } 36 37 static int __pagemap_scan_get_categories(int fd, char *start, struct page_region *r) 38 { 39 struct pm_scan_arg arg; 40 41 arg.start = (uintptr_t)start; 42 arg.end = (uintptr_t)(start + psize()); 43 arg.vec = (uintptr_t)r; 44 arg.vec_len = 1; 45 arg.flags = 0; 46 arg.size = sizeof(struct pm_scan_arg); 47 arg.max_pages = 0; 48 arg.category_inverted = 0; 49 arg.category_mask = 0; 50 arg.category_anyof_mask = PAGE_IS_WPALLOWED | PAGE_IS_WRITTEN | PAGE_IS_FILE | 51 PAGE_IS_PRESENT | PAGE_IS_SWAPPED | PAGE_IS_PFNZERO | 52 PAGE_IS_HUGE | PAGE_IS_SOFT_DIRTY; 53 arg.return_mask = arg.category_anyof_mask; 54 55 return ioctl(fd, PAGEMAP_SCAN, &arg); 56 } 57 58 static uint64_t pagemap_scan_get_categories(int fd, char *start) 59 { 60 struct page_region r; 61 int ret; 62 63 ret = __pagemap_scan_get_categories(fd, start, &r); 64 if (ret < 0) 65 ksft_exit_fail_msg("PAGEMAP_SCAN failed: %s\n", strerror(errno)); 66 if (ret == 0) 67 return 0; 68 return r.categories; 69 } 70 71 /* `start` is any valid address. */ 72 static bool pagemap_scan_supported(int fd, char *start) 73 { 74 static int supported = -1; 75 int ret; 76 77 if (supported != -1) 78 return supported; 79 80 /* Provide an invalid address in order to trigger EFAULT. */ 81 ret = __pagemap_scan_get_categories(fd, start, (struct page_region *) ~0UL); 82 if (ret == 0) 83 ksft_exit_fail_msg("PAGEMAP_SCAN succeeded unexpectedly\n"); 84 85 supported = errno == EFAULT; 86 87 return supported; 88 } 89 90 static bool page_entry_is(int fd, char *start, char *desc, 91 uint64_t pagemap_flags, uint64_t pagescan_flags) 92 { 93 bool m = pagemap_get_entry(fd, start) & pagemap_flags; 94 95 if (pagemap_scan_supported(fd, start)) { 96 bool s = pagemap_scan_get_categories(fd, start) & pagescan_flags; 97 98 if (m == s) 99 return m; 100 101 ksft_exit_fail_msg( 102 "read and ioctl return unmatched results for %s: %d %d", desc, m, s); 103 } 104 return m; 105 } 106 107 bool pagemap_is_softdirty(int fd, char *start) 108 { 109 return page_entry_is(fd, start, "soft-dirty", 110 PM_SOFT_DIRTY, PAGE_IS_SOFT_DIRTY); 111 } 112 113 bool pagemap_is_swapped(int fd, char *start) 114 { 115 return page_entry_is(fd, start, "swap", PM_SWAP, PAGE_IS_SWAPPED); 116 } 117 118 bool pagemap_is_populated(int fd, char *start) 119 { 120 return page_entry_is(fd, start, "populated", 121 PM_PRESENT | PM_SWAP, 122 PAGE_IS_PRESENT | PAGE_IS_SWAPPED); 123 } 124 125 unsigned long pagemap_get_pfn(int fd, char *start) 126 { 127 uint64_t entry = pagemap_get_entry(fd, start); 128 129 /* If present (63th bit), PFN is at bit 0 -- 54. */ 130 if (entry & PM_PRESENT) 131 return entry & 0x007fffffffffffffull; 132 return -1ul; 133 } 134 135 void clear_softdirty(void) 136 { 137 int ret; 138 const char *ctrl = "4"; 139 int fd = open("/proc/self/clear_refs", O_WRONLY); 140 141 if (fd < 0) 142 ksft_exit_fail_msg("opening clear_refs failed\n"); 143 ret = write(fd, ctrl, strlen(ctrl)); 144 close(fd); 145 if (ret != (signed int)strlen(ctrl)) 146 ksft_exit_fail_msg("writing clear_refs failed\n"); 147 } 148 149 bool check_for_pattern(FILE *fp, const char *pattern, char *buf, size_t len) 150 { 151 while (fgets(buf, len, fp)) { 152 if (!strncmp(buf, pattern, strlen(pattern))) 153 return true; 154 } 155 return false; 156 } 157 158 uint64_t read_pmd_pagesize(void) 159 { 160 int fd; 161 char buf[20]; 162 ssize_t num_read; 163 164 fd = open(PMD_SIZE_FILE_PATH, O_RDONLY); 165 if (fd == -1) 166 return 0; 167 168 num_read = read(fd, buf, 19); 169 if (num_read < 1) { 170 close(fd); 171 return 0; 172 } 173 buf[num_read] = '\0'; 174 close(fd); 175 176 return strtoul(buf, NULL, 10); 177 } 178 179 unsigned long rss_anon(void) 180 { 181 unsigned long rss_anon = 0; 182 FILE *fp; 183 char buffer[MAX_LINE_LENGTH]; 184 185 fp = fopen(STATUS_FILE_PATH, "r"); 186 if (!fp) 187 ksft_exit_fail_msg("%s: Failed to open file %s\n", __func__, STATUS_FILE_PATH); 188 189 if (!check_for_pattern(fp, "RssAnon:", buffer, sizeof(buffer))) 190 goto err_out; 191 192 if (sscanf(buffer, "RssAnon:%10lu kB", &rss_anon) != 1) 193 ksft_exit_fail_msg("Reading status error\n"); 194 195 err_out: 196 fclose(fp); 197 return rss_anon; 198 } 199 200 static int vaddr_pageflags_get(char *vaddr, int pagemap_fd, int kpageflags_fd, 201 uint64_t *flags) 202 { 203 unsigned long pfn; 204 205 pfn = pagemap_get_pfn(pagemap_fd, vaddr); 206 207 /* non-present PFN */ 208 if (pfn == -1UL) 209 return 1; 210 211 if (pageflags_get(pfn, kpageflags_fd, flags)) 212 return -1; 213 214 return 0; 215 } 216 217 /* 218 * gather_folio_orders - scan through [vaddr_start, len) and record 219 * folio orders 220 * 221 * @vaddr_start: start vaddr 222 * @len: range length 223 * @pagemap_fd: file descriptor to /proc/<pid>/pagemap 224 * @kpageflags_fd: file descriptor to /proc/kpageflags 225 * @orders: output folio order array 226 * @nr_orders: folio order array size 227 * 228 * gather_folio_orders() scan through [vaddr_start, len) and check 229 * all folios within the range and record their orders. All order-0 pages will 230 * be recorded. Non-present vaddr is skipped. 231 * 232 * Return: 0 - no error, -1 - unhandled cases 233 */ 234 int gather_folio_orders(char *vaddr_start, size_t len, 235 int pagemap_fd, int kpageflags_fd, int orders[], int nr_orders) 236 { 237 uint64_t page_flags = 0; 238 int cur_order = -1; 239 char *vaddr; 240 241 if (pagemap_fd == -1 || kpageflags_fd == -1) 242 return -1; 243 if (!orders) 244 return -1; 245 if (nr_orders <= 0) 246 return -1; 247 248 for (vaddr = vaddr_start; vaddr < vaddr_start + len;) { 249 char *next_folio_vaddr; 250 int status; 251 252 status = vaddr_pageflags_get(vaddr, pagemap_fd, kpageflags_fd, 253 &page_flags); 254 if (status < 0) 255 return -1; 256 257 /* skip non present vaddr */ 258 if (status == 1) { 259 vaddr += psize(); 260 continue; 261 } 262 263 /* all order-0 pages with possible false postive (non folio) */ 264 if (!(page_flags & (KPF_COMPOUND_HEAD | KPF_COMPOUND_TAIL))) { 265 orders[0]++; 266 vaddr += psize(); 267 continue; 268 } 269 270 /* skip non thp compound pages */ 271 if (!(page_flags & KPF_THP)) { 272 vaddr += psize(); 273 continue; 274 } 275 276 /* vpn points to part of a THP at this point */ 277 if (page_flags & KPF_COMPOUND_HEAD) 278 cur_order = 1; 279 else { 280 vaddr += psize(); 281 continue; 282 } 283 284 next_folio_vaddr = vaddr + (1UL << (cur_order + pshift())); 285 286 if (next_folio_vaddr >= vaddr_start + len) 287 break; 288 289 while ((status = vaddr_pageflags_get(next_folio_vaddr, 290 pagemap_fd, kpageflags_fd, 291 &page_flags)) >= 0) { 292 /* 293 * non present vaddr, next compound head page, or 294 * order-0 page 295 */ 296 if (status == 1 || 297 (page_flags & KPF_COMPOUND_HEAD) || 298 !(page_flags & (KPF_COMPOUND_HEAD | KPF_COMPOUND_TAIL))) { 299 if (cur_order < nr_orders) { 300 orders[cur_order]++; 301 cur_order = -1; 302 vaddr = next_folio_vaddr; 303 } 304 break; 305 } 306 307 cur_order++; 308 next_folio_vaddr = vaddr + (1UL << (cur_order + pshift())); 309 } 310 311 if (status < 0) 312 return status; 313 } 314 if (cur_order > 0 && cur_order < nr_orders) 315 orders[cur_order]++; 316 return 0; 317 } 318 319 char *__get_smap_entry(void *addr, const char *pattern, char *buf, size_t len) 320 { 321 int ret; 322 FILE *fp; 323 char *entry = NULL; 324 char addr_pattern[MAX_LINE_LENGTH]; 325 326 ret = snprintf(addr_pattern, MAX_LINE_LENGTH, "%08lx-", 327 (unsigned long) addr); 328 if (ret >= MAX_LINE_LENGTH) 329 ksft_exit_fail_msg("%s: Pattern is too long\n", __func__); 330 331 fp = fopen(SMAP_FILE_PATH, "r"); 332 if (!fp) 333 ksft_exit_fail_msg("%s: Failed to open file %s\n", __func__, SMAP_FILE_PATH); 334 335 if (!check_for_pattern(fp, addr_pattern, buf, len)) 336 goto err_out; 337 338 /* Fetch the pattern in the same block */ 339 if (!check_for_pattern(fp, pattern, buf, len)) 340 goto err_out; 341 342 /* Trim trailing newline */ 343 entry = strchr(buf, '\n'); 344 if (entry) 345 *entry = '\0'; 346 347 entry = buf + strlen(pattern); 348 349 err_out: 350 fclose(fp); 351 return entry; 352 } 353 354 static bool __check_pmd_huge(void *addr, char *pattern, int nr_hpages, 355 uint64_t hpage_size) 356 { 357 char buffer[MAX_LINE_LENGTH]; 358 uint64_t thp = -1; 359 char *entry; 360 361 entry = __get_smap_entry(addr, pattern, buffer, sizeof(buffer)); 362 if (!entry) 363 goto err_out; 364 365 if (sscanf(entry, "%9" SCNu64 " kB", &thp) != 1) 366 ksft_exit_fail_msg("Reading smap error\n"); 367 368 err_out: 369 return thp == (nr_hpages * (hpage_size >> 10)); 370 } 371 372 static bool check_large_folios(void *addr, size_t len, int nr_hpages, 373 uint64_t hpage_size) 374 { 375 int order = 0, pagesize = getpagesize(); 376 unsigned int nr_pages = hpage_size / pagesize; 377 int orders[MAX_NR_ORDERS], status; 378 int pagemap_fd, kpageflags_fd; 379 bool ret = false; 380 381 if (!nr_pages) 382 ksft_exit_fail_msg("invalid hugepage size\n"); 383 384 order = 31 - __builtin_clz(nr_pages); 385 if (!order || order >= MAX_NR_ORDERS) 386 ksft_exit_fail_msg("invalid order\n"); 387 388 memset(orders, 0, sizeof(int) * MAX_NR_ORDERS); 389 pagemap_fd = open(PAGEMAP_PATH, O_RDONLY); 390 if (pagemap_fd == -1) 391 ksft_exit_fail_msg("read pagemap fail\n"); 392 393 kpageflags_fd = open(KPAGEFLAGS_PATH, O_RDONLY); 394 if (kpageflags_fd == -1) { 395 close(pagemap_fd); 396 ksft_exit_fail_msg("read kpageflags fail\n"); 397 } 398 399 status = gather_folio_orders(addr, len, pagemap_fd, 400 kpageflags_fd, orders, MAX_NR_ORDERS); 401 if (status) 402 goto out; 403 404 if (orders[order] == nr_hpages) 405 ret = true; 406 407 out: 408 close(pagemap_fd); 409 close(kpageflags_fd); 410 return ret; 411 } 412 413 bool check_huge_anon(void *addr, size_t len, int nr_hpages, uint64_t hpage_size) 414 { 415 uint64_t pmd_pagesize = read_pmd_pagesize(); 416 417 if (!pmd_pagesize) 418 ksft_exit_fail_msg("reading PMD pagesize failed\n"); 419 420 if (hpage_size == pmd_pagesize) 421 return __check_pmd_huge(addr, "AnonHugePages: ", nr_hpages, hpage_size); 422 423 return check_large_folios(addr, len, nr_hpages, hpage_size); 424 } 425 426 bool check_huge_file(void *addr, size_t len, int nr_hpages, uint64_t hpage_size) 427 { 428 uint64_t pmd_pagesize = read_pmd_pagesize(); 429 430 if (!pmd_pagesize) 431 ksft_exit_fail_msg("reading PMD pagesize failed\n"); 432 433 if (hpage_size == pmd_pagesize) 434 return __check_pmd_huge(addr, "FilePmdMapped:", nr_hpages, hpage_size); 435 436 return check_large_folios(addr, len, nr_hpages, hpage_size); 437 } 438 439 bool check_huge_shmem(void *addr, size_t len, int nr_hpages, uint64_t hpage_size) 440 { 441 uint64_t pmd_pagesize = read_pmd_pagesize(); 442 443 if (!pmd_pagesize) 444 ksft_exit_fail_msg("reading PMD pagesize failed\n"); 445 446 if (hpage_size == pmd_pagesize) 447 return __check_pmd_huge(addr, "ShmemPmdMapped:", nr_hpages, hpage_size); 448 449 return check_large_folios(addr, len, nr_hpages, hpage_size); 450 } 451 452 int64_t allocate_transhuge(void *ptr, int pagemap_fd) 453 { 454 uint64_t ent[2]; 455 456 /* drop pmd */ 457 if (mmap(ptr, HPAGE_SIZE, PROT_READ | PROT_WRITE, 458 MAP_FIXED | MAP_ANONYMOUS | 459 MAP_NORESERVE | MAP_PRIVATE, -1, 0) != ptr) 460 ksft_exit_fail_msg("mmap transhuge\n"); 461 462 if (madvise(ptr, HPAGE_SIZE, MADV_HUGEPAGE)) 463 ksft_exit_fail_msg("MADV_HUGEPAGE\n"); 464 465 /* allocate transparent huge page */ 466 *(volatile void **)ptr = ptr; 467 468 if (pread(pagemap_fd, ent, sizeof(ent), 469 (uintptr_t)ptr >> (pshift() - 3)) != sizeof(ent)) 470 ksft_exit_fail_msg("read pagemap\n"); 471 472 if (PAGEMAP_PRESENT(ent[0]) && PAGEMAP_PRESENT(ent[1]) && 473 PAGEMAP_PFN(ent[0]) + 1 == PAGEMAP_PFN(ent[1]) && 474 !(PAGEMAP_PFN(ent[0]) & ((1 << (HPAGE_SHIFT - pshift())) - 1))) 475 return PAGEMAP_PFN(ent[0]); 476 477 return -1; 478 } 479 480 int pageflags_get(unsigned long pfn, int kpageflags_fd, uint64_t *flags) 481 { 482 size_t count; 483 484 count = pread(kpageflags_fd, flags, sizeof(*flags), 485 pfn * sizeof(*flags)); 486 487 if (count != sizeof(*flags)) 488 return -1; 489 490 return 0; 491 } 492 493 bool is_backed_by_folio(char *vaddr, int order, int pagemap_fd, 494 int kpageflags_fd) 495 { 496 const uint64_t folio_head_flags = KPF_THP | KPF_COMPOUND_HEAD; 497 const uint64_t folio_tail_flags = KPF_THP | KPF_COMPOUND_TAIL; 498 const unsigned long nr_pages = 1UL << order; 499 unsigned long pfn_head; 500 uint64_t pfn_flags; 501 unsigned long pfn; 502 unsigned long i; 503 504 pfn = pagemap_get_pfn(pagemap_fd, vaddr); 505 506 /* non present page */ 507 if (pfn == -1UL) 508 return false; 509 510 if (pageflags_get(pfn, kpageflags_fd, &pfn_flags)) 511 goto fail; 512 513 /* check for order-0 pages */ 514 if (!order) { 515 if (pfn_flags & (folio_head_flags | folio_tail_flags)) 516 return false; 517 return true; 518 } 519 520 /* non THP folio */ 521 if (!(pfn_flags & KPF_THP)) 522 return false; 523 524 pfn_head = pfn & ~(nr_pages - 1); 525 526 if (pageflags_get(pfn_head, kpageflags_fd, &pfn_flags)) 527 goto fail; 528 529 /* head PFN has no compound_head flag set */ 530 if ((pfn_flags & folio_head_flags) != folio_head_flags) 531 return false; 532 533 /* check all tail PFN flags */ 534 for (i = 1; i < nr_pages; i++) { 535 if (pageflags_get(pfn_head + i, kpageflags_fd, &pfn_flags)) 536 goto fail; 537 if ((pfn_flags & folio_tail_flags) != folio_tail_flags) 538 return false; 539 } 540 541 /* 542 * check the PFN after this folio, but if its flags cannot be obtained, 543 * assume this folio has the expected order 544 */ 545 if (pageflags_get(pfn_head + nr_pages, kpageflags_fd, &pfn_flags)) 546 return true; 547 548 /* If we find another tail page, then the folio is larger. */ 549 return (pfn_flags & folio_tail_flags) != folio_tail_flags; 550 fail: 551 ksft_exit_fail_msg("Failed to get folio info\n"); 552 } 553 554 /** 555 * is_range_backed_by_order() - check that a range is backed by @order folios 556 * @start: start of the range, a multiple of the folio size 557 * @len: length of the range in bytes, a multiple of the folio size 558 * @order: the folio order to check for 559 * @pagemap_fd: open /proc/<pid>/pagemap of the range's owner 560 * @kpageflags_fd: open /proc/kpageflags 561 * 562 * Every folio-sized, folio-aligned part of the range must map one folio of 563 * @order, head to tail, with the head at the start of the part. A part 564 * backed by several smaller folios fails, and so does a folio mapped off 565 * its natural alignment. 566 * 567 * Returns: true if the whole range is backed that way, false otherwise. 568 */ 569 bool is_range_backed_by_order(char *start, size_t len, int order, 570 int pagemap_fd, int kpageflags_fd) 571 { 572 const unsigned long nr_pages = 1UL << order; 573 const size_t folio_size = nr_pages * psize(); 574 char *vaddr; 575 576 if ((uintptr_t)start % folio_size || len % folio_size) 577 return false; 578 579 for (vaddr = start; vaddr < start + len; vaddr += folio_size) { 580 const unsigned long pfn = pagemap_get_pfn(pagemap_fd, vaddr); 581 unsigned long i; 582 583 /* Not present, or a tail page */ 584 if (pfn == -1UL || pfn % nr_pages) 585 return false; 586 587 for (i = 1; i < nr_pages; i++) { 588 char *page = vaddr + i * psize(); 589 590 if (pagemap_get_pfn(pagemap_fd, page) != pfn + i) 591 return false; 592 } 593 594 if (!is_backed_by_folio(vaddr, order, pagemap_fd, kpageflags_fd)) 595 return false; 596 } 597 598 return true; 599 } 600 601 #define TRACEFS_ROOT "/sys/kernel/tracing" 602 603 /* 604 * Returns -1 without tracefs or the subsystem. The events are system-wide: 605 * whoever switches them on has to switch them off again, on every exit path. 606 */ 607 int tracing_events_open(const char *subsys) 608 { 609 char path[256]; 610 611 snprintf(path, sizeof(path), TRACEFS_ROOT "/events/%s/enable", 612 subsys); 613 return open(path, O_WRONLY); 614 } 615 616 int tracing_events_enable(int fd, bool enable) 617 { 618 if (pwrite(fd, enable ? "1" : "0", 1, 0) != 1) 619 return -1; 620 return 0; 621 } 622 623 /* Drop what the trace buffer holds so far */ 624 int tracing_clear_trace(void) 625 { 626 int fd = open(TRACEFS_ROOT "/trace", O_WRONLY | O_TRUNC); 627 628 if (fd < 0) 629 return -1; 630 close(fd); 631 return 0; 632 } 633 634 FILE *tracing_open_trace(void) 635 { 636 return fopen(TRACEFS_ROOT "/trace", "r"); 637 } 638 639 /* If `ioctls' non-NULL, the allowed ioctls will be returned into the var */ 640 int uffd_register_with_ioctls(int uffd, void *addr, uint64_t len, 641 bool miss, bool wp, bool minor, uint64_t *ioctls) 642 { 643 struct uffdio_register uffdio_register = { 0 }; 644 uint64_t mode = 0; 645 int ret = 0; 646 647 if (miss) 648 mode |= UFFDIO_REGISTER_MODE_MISSING; 649 if (wp) 650 mode |= UFFDIO_REGISTER_MODE_WP; 651 if (minor) 652 mode |= UFFDIO_REGISTER_MODE_MINOR; 653 654 uffdio_register.range.start = (unsigned long)addr; 655 uffdio_register.range.len = len; 656 uffdio_register.mode = mode; 657 658 if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register) == -1) 659 ret = -errno; 660 else if (ioctls) 661 *ioctls = uffdio_register.ioctls; 662 663 return ret; 664 } 665 666 int uffd_register(int uffd, void *addr, uint64_t len, 667 bool miss, bool wp, bool minor) 668 { 669 return uffd_register_with_ioctls(uffd, addr, len, 670 miss, wp, minor, NULL); 671 } 672 673 int uffd_unregister(int uffd, void *addr, uint64_t len) 674 { 675 struct uffdio_range range = { .start = (uintptr_t)addr, .len = len }; 676 int ret = 0; 677 678 if (ioctl(uffd, UFFDIO_UNREGISTER, &range) == -1) 679 ret = -errno; 680 681 return ret; 682 } 683 684 static bool check_vmflag(void *addr, const char *flag) 685 { 686 char buffer[MAX_LINE_LENGTH]; 687 const char *flags; 688 size_t flaglen; 689 690 flags = __get_smap_entry(addr, "VmFlags:", buffer, sizeof(buffer)); 691 if (!flags) 692 ksft_exit_fail_msg("%s: No VmFlags for %p\n", __func__, addr); 693 694 while (true) { 695 flags += strspn(flags, " "); 696 697 flaglen = strcspn(flags, " "); 698 if (!flaglen) 699 return false; 700 701 if (flaglen == strlen(flag) && !memcmp(flags, flag, flaglen)) 702 return true; 703 704 flags += flaglen; 705 } 706 } 707 708 bool check_vmflag_io(void *addr) 709 { 710 return check_vmflag(addr, "io"); 711 } 712 713 bool check_vmflag_pfnmap(void *addr) 714 { 715 return check_vmflag(addr, "pf"); 716 } 717 718 bool check_vmflag_guard(void *addr) 719 { 720 return check_vmflag(addr, "gu"); 721 } 722 723 bool softdirty_supported(void) 724 { 725 char *addr; 726 bool supported = false; 727 const size_t pagesize = getpagesize(); 728 729 /* New mappings are expected to be marked with VM_SOFTDIRTY (sd). */ 730 addr = mmap(0, pagesize, PROT_READ | PROT_WRITE, 731 MAP_ANONYMOUS | MAP_PRIVATE, 0, 0); 732 if (addr == MAP_FAILED) 733 ksft_exit_fail_msg("mmap failed\n"); 734 735 supported = check_vmflag(addr, "sd"); 736 munmap(addr, pagesize); 737 return supported; 738 } 739 740 /* 741 * Open an fd at /proc/$pid/maps and configure procmap_out ready for 742 * PROCMAP_QUERY query. Returns 0 on success, or an error code otherwise. 743 */ 744 int open_procmap(pid_t pid, struct procmap_fd *procmap_out) 745 { 746 char path[256]; 747 int ret = 0; 748 749 memset(procmap_out, '\0', sizeof(*procmap_out)); 750 sprintf(path, "/proc/%d/maps", pid); 751 procmap_out->query.size = sizeof(procmap_out->query); 752 procmap_out->fd = open(path, O_RDONLY); 753 if (procmap_out->fd < 0) 754 ret = -errno; 755 756 return ret; 757 } 758 759 /* Perform PROCMAP_QUERY. Returns 0 on success, or an error code otherwise. */ 760 int query_procmap(struct procmap_fd *procmap) 761 { 762 int ret = 0; 763 764 if (ioctl(procmap->fd, PROCMAP_QUERY, &procmap->query) == -1) 765 ret = -errno; 766 767 return ret; 768 } 769 770 /* 771 * Try to find the VMA at specified address, returns true if found, false if not 772 * found, and the test is failed if any other error occurs. 773 * 774 * On success, procmap->query is populated with the results. 775 */ 776 bool find_vma_procmap(struct procmap_fd *procmap, void *address) 777 { 778 int err; 779 780 procmap->query.query_flags = 0; 781 procmap->query.query_addr = (unsigned long)address; 782 err = query_procmap(procmap); 783 if (!err) 784 return true; 785 786 if (err != -ENOENT) 787 ksft_exit_fail_msg("%s: Error %d on ioctl(PROCMAP_QUERY)\n", 788 __func__, err); 789 return false; 790 } 791 792 /* 793 * Close fd used by PROCMAP_QUERY mechanism. Returns 0 on success, or an error 794 * code otherwise. 795 */ 796 int close_procmap(struct procmap_fd *procmap) 797 { 798 return close(procmap->fd); 799 } 800 801 int write_sysfs(const char *file_path, unsigned long val) 802 { 803 FILE *f = fopen(file_path, "w"); 804 805 if (!f) { 806 fprintf(stderr, "f %s\n", file_path); 807 perror("fopen"); 808 return 1; 809 } 810 if (fprintf(f, "%lu", val) < 0) { 811 perror("fprintf"); 812 fclose(f); 813 return 1; 814 } 815 fclose(f); 816 817 return 0; 818 } 819 820 int read_sysfs(const char *file_path, unsigned long *val) 821 { 822 FILE *f = fopen(file_path, "r"); 823 824 if (!f) { 825 fprintf(stderr, "f %s\n", file_path); 826 perror("fopen"); 827 return 1; 828 } 829 if (fscanf(f, "%lu", val) != 1) { 830 perror("fscanf"); 831 fclose(f); 832 return 1; 833 } 834 fclose(f); 835 836 return 0; 837 } 838 839 void *sys_mremap(void *old_address, unsigned long old_size, 840 unsigned long new_size, int flags, void *new_address) 841 { 842 return (void *)syscall(__NR_mremap, (unsigned long)old_address, 843 old_size, new_size, flags, 844 (unsigned long)new_address); 845 } 846 847 bool detect_huge_zeropage(void) 848 { 849 int fd = open("/sys/kernel/mm/transparent_hugepage/use_zero_page", 850 O_RDONLY); 851 bool enabled = 0; 852 char buf[15]; 853 int ret; 854 855 if (fd < 0) 856 return 0; 857 858 ret = pread(fd, buf, sizeof(buf), 0); 859 if (ret > 0 && ret < sizeof(buf)) { 860 buf[ret] = 0; 861 862 if (strtoul(buf, NULL, 10) == 1) 863 enabled = 1; 864 } 865 866 close(fd); 867 return enabled; 868 } 869 870 long ksm_get_self_zero_pages(void) 871 { 872 int proc_self_ksm_stat_fd; 873 char buf[200]; 874 char *substr_ksm_zero; 875 size_t value_pos; 876 ssize_t read_size; 877 878 proc_self_ksm_stat_fd = open("/proc/self/ksm_stat", O_RDONLY); 879 if (proc_self_ksm_stat_fd < 0) 880 return -errno; 881 882 read_size = pread(proc_self_ksm_stat_fd, buf, sizeof(buf) - 1, 0); 883 close(proc_self_ksm_stat_fd); 884 if (read_size < 0) 885 return -errno; 886 887 buf[read_size] = 0; 888 889 substr_ksm_zero = strstr(buf, "ksm_zero_pages"); 890 if (!substr_ksm_zero) 891 return 0; 892 893 value_pos = strcspn(substr_ksm_zero, "0123456789"); 894 return strtol(substr_ksm_zero + value_pos, NULL, 10); 895 } 896 897 long ksm_get_self_merging_pages(void) 898 { 899 int proc_self_ksm_merging_pages_fd; 900 char buf[10]; 901 ssize_t ret; 902 903 proc_self_ksm_merging_pages_fd = open("/proc/self/ksm_merging_pages", 904 O_RDONLY); 905 if (proc_self_ksm_merging_pages_fd < 0) 906 return -errno; 907 908 ret = pread(proc_self_ksm_merging_pages_fd, buf, sizeof(buf) - 1, 0); 909 close(proc_self_ksm_merging_pages_fd); 910 if (ret <= 0) 911 return -errno; 912 buf[ret] = 0; 913 914 return strtol(buf, NULL, 10); 915 } 916 917 long ksm_get_full_scans(void) 918 { 919 int ksm_full_scans_fd; 920 char buf[10]; 921 ssize_t ret; 922 923 ksm_full_scans_fd = open("/sys/kernel/mm/ksm/full_scans", O_RDONLY); 924 if (ksm_full_scans_fd < 0) 925 return -errno; 926 927 ret = pread(ksm_full_scans_fd, buf, sizeof(buf) - 1, 0); 928 close(ksm_full_scans_fd); 929 if (ret <= 0) 930 return -errno; 931 buf[ret] = 0; 932 933 return strtol(buf, NULL, 10); 934 } 935 936 int ksm_use_zero_pages(void) 937 { 938 int ksm_use_zero_pages_fd; 939 ssize_t ret; 940 941 ksm_use_zero_pages_fd = open("/sys/kernel/mm/ksm/use_zero_pages", O_RDWR); 942 if (ksm_use_zero_pages_fd < 0) 943 return -errno; 944 945 ret = write(ksm_use_zero_pages_fd, "1", 1); 946 close(ksm_use_zero_pages_fd); 947 return ret == 1 ? 0 : -errno; 948 } 949 950 int ksm_start(void) 951 { 952 int ksm_fd; 953 ssize_t ret; 954 long start_scans, end_scans; 955 956 ksm_fd = open("/sys/kernel/mm/ksm/run", O_RDWR); 957 if (ksm_fd < 0) 958 return -errno; 959 960 /* Wait for two full scans such that any possible merging happened. */ 961 start_scans = ksm_get_full_scans(); 962 if (start_scans < 0) { 963 close(ksm_fd); 964 return start_scans; 965 } 966 ret = write(ksm_fd, "1", 1); 967 close(ksm_fd); 968 if (ret != 1) 969 return -errno; 970 do { 971 end_scans = ksm_get_full_scans(); 972 if (end_scans < 0) 973 return end_scans; 974 } while (end_scans < start_scans + 2); 975 976 return 0; 977 } 978 979 int ksm_stop(void) 980 { 981 int ksm_fd; 982 ssize_t ret; 983 984 ksm_fd = open("/sys/kernel/mm/ksm/run", O_RDWR); 985 if (ksm_fd < 0) 986 return -errno; 987 988 ret = write(ksm_fd, "2", 1); 989 close(ksm_fd); 990 return ret == 1 ? 0 : -errno; 991 } 992 993 int get_hardware_corrupted_size(unsigned long *val) 994 { 995 unsigned long size; 996 char *line = NULL; 997 size_t linelen = 0; 998 FILE *f = fopen("/proc/meminfo", "r"); 999 int ret = -1; 1000 1001 if (!f) 1002 return ret; 1003 1004 while (getline(&line, &linelen, f) > 0) { 1005 if (sscanf(line, "HardwareCorrupted: %12lu kB", &size) == 1) { 1006 *val = size; 1007 ret = 0; 1008 break; 1009 } 1010 } 1011 1012 free(line); 1013 fclose(f); 1014 return ret; 1015 } 1016 1017 int unpoison_memory(unsigned long pfn) 1018 { 1019 int unpoison_fd, len; 1020 char buf[32]; 1021 ssize_t ret; 1022 1023 unpoison_fd = open("/sys/kernel/debug/hwpoison/unpoison-pfn", O_WRONLY); 1024 if (unpoison_fd < 0) 1025 return -errno; 1026 1027 len = sprintf(buf, "0x%lx\n", pfn); 1028 ret = write(unpoison_fd, buf, len); 1029 close(unpoison_fd); 1030 1031 return ret > 0 ? 0 : -errno; 1032 } 1033 1034 static unsigned long shmall, shmmax; 1035 1036 void __shm_limits_restore(void) 1037 { 1038 int ret; 1039 1040 if (shmmax) { 1041 ret = write_num("/proc/sys/kernel/shmmax", shmmax); 1042 if (ret < 0) 1043 ksft_exit_fail_msg("Failed to restore shmmax: %s\n", 1044 strerror(-ret)); 1045 } 1046 if (shmall) { 1047 ret = write_num("/proc/sys/kernel/shmall", shmall); 1048 if (ret < 0) 1049 ksft_exit_fail_msg("Failed to restore shmall: %s\n", 1050 strerror(-ret)); 1051 } 1052 } 1053 1054 void shm_limits_prepare(unsigned long length) 1055 { 1056 unsigned long nr = length / psize(); 1057 unsigned long val; 1058 int ret; 1059 1060 ret = read_num("/proc/sys/kernel/shmmax", &val); 1061 if (ret < 0) 1062 ksft_exit_fail_msg("Failed to read /proc/sys/kernel/shmmax: %s\n", 1063 strerror(-ret)); 1064 1065 if (val < length) { 1066 ret = write_num("/proc/sys/kernel/shmmax", length); 1067 if (ret < 0) 1068 ksft_exit_fail_msg("Failed to write %lu to /proc/sys/kernel/shmmax: %s\n", 1069 length, strerror(-ret)); 1070 shmmax = val; 1071 } 1072 1073 ret = read_num("/proc/sys/kernel/shmall", &val); 1074 if (ret < 0) 1075 ksft_exit_fail_msg("Failed to read /proc/sys/kernel/shmall: %s\n", 1076 strerror(-ret)); 1077 if (val < nr) { 1078 ret = write_num("/proc/sys/kernel/shmall", nr); 1079 if (ret < 0) 1080 ksft_exit_fail_msg("Failed to write %lu to /proc/sys/kernel/shmall: %s\n", 1081 nr, strerror(-ret)); 1082 shmall = val; 1083 } 1084 } 1085