1 // SPDX-License-Identifier: GPL-2.0 2 #include <string.h> 3 #include <errno.h> 4 #include <fcntl.h> 5 #include <inttypes.h> 6 #include <sys/ioctl.h> 7 #include <linux/userfaultfd.h> 8 #include <linux/fs.h> 9 #include <sys/syscall.h> 10 #include <unistd.h> 11 #include "kselftest.h" 12 #include "vm_util.h" 13 14 #define PMD_SIZE_FILE_PATH "/sys/kernel/mm/transparent_hugepage/hpage_pmd_size" 15 #define SMAP_FILE_PATH "/proc/self/smaps" 16 #define STATUS_FILE_PATH "/proc/self/status" 17 #define MAX_LINE_LENGTH 500 18 #define PAGEMAP_PATH "/proc/self/pagemap" 19 #define KPAGEFLAGS_PATH "/proc/kpageflags" 20 #define MAX_NR_ORDERS 20 21 22 unsigned int __page_size; 23 unsigned int __page_shift; 24 25 uint64_t pagemap_get_entry(int fd, char *start) 26 { 27 const unsigned long pfn = (unsigned long)start / getpagesize(); 28 uint64_t entry; 29 int ret; 30 31 ret = pread(fd, &entry, sizeof(entry), pfn * sizeof(entry)); 32 if (ret != sizeof(entry)) 33 ksft_exit_fail_msg("reading pagemap failed\n"); 34 return entry; 35 } 36 37 static int __pagemap_scan_get_categories(int fd, char *start, struct page_region *r) 38 { 39 struct pm_scan_arg arg; 40 41 arg.start = (uintptr_t)start; 42 arg.end = (uintptr_t)(start + psize()); 43 arg.vec = (uintptr_t)r; 44 arg.vec_len = 1; 45 arg.flags = 0; 46 arg.size = sizeof(struct pm_scan_arg); 47 arg.max_pages = 0; 48 arg.category_inverted = 0; 49 arg.category_mask = 0; 50 arg.category_anyof_mask = PAGE_IS_WPALLOWED | PAGE_IS_WRITTEN | PAGE_IS_FILE | 51 PAGE_IS_PRESENT | PAGE_IS_SWAPPED | PAGE_IS_PFNZERO | 52 PAGE_IS_HUGE | PAGE_IS_SOFT_DIRTY; 53 arg.return_mask = arg.category_anyof_mask; 54 55 return ioctl(fd, PAGEMAP_SCAN, &arg); 56 } 57 58 static uint64_t pagemap_scan_get_categories(int fd, char *start) 59 { 60 struct page_region r; 61 int ret; 62 63 ret = __pagemap_scan_get_categories(fd, start, &r); 64 if (ret < 0) 65 ksft_exit_fail_msg("PAGEMAP_SCAN failed: %s\n", strerror(errno)); 66 if (ret == 0) 67 return 0; 68 return r.categories; 69 } 70 71 /* `start` is any valid address. */ 72 static bool pagemap_scan_supported(int fd, char *start) 73 { 74 static int supported = -1; 75 int ret; 76 77 if (supported != -1) 78 return supported; 79 80 /* Provide an invalid address in order to trigger EFAULT. */ 81 ret = __pagemap_scan_get_categories(fd, start, (struct page_region *) ~0UL); 82 if (ret == 0) 83 ksft_exit_fail_msg("PAGEMAP_SCAN succeeded unexpectedly\n"); 84 85 supported = errno == EFAULT; 86 87 return supported; 88 } 89 90 static bool page_entry_is(int fd, char *start, char *desc, 91 uint64_t pagemap_flags, uint64_t pagescan_flags) 92 { 93 bool m = pagemap_get_entry(fd, start) & pagemap_flags; 94 95 if (pagemap_scan_supported(fd, start)) { 96 bool s = pagemap_scan_get_categories(fd, start) & pagescan_flags; 97 98 if (m == s) 99 return m; 100 101 ksft_exit_fail_msg( 102 "read and ioctl return unmatched results for %s: %d %d", desc, m, s); 103 } 104 return m; 105 } 106 107 bool pagemap_is_softdirty(int fd, char *start) 108 { 109 return page_entry_is(fd, start, "soft-dirty", 110 PM_SOFT_DIRTY, PAGE_IS_SOFT_DIRTY); 111 } 112 113 bool pagemap_is_swapped(int fd, char *start) 114 { 115 return page_entry_is(fd, start, "swap", PM_SWAP, PAGE_IS_SWAPPED); 116 } 117 118 bool pagemap_is_populated(int fd, char *start) 119 { 120 return page_entry_is(fd, start, "populated", 121 PM_PRESENT | PM_SWAP, 122 PAGE_IS_PRESENT | PAGE_IS_SWAPPED); 123 } 124 125 unsigned long pagemap_get_pfn(int fd, char *start) 126 { 127 uint64_t entry = pagemap_get_entry(fd, start); 128 129 /* If present (63th bit), PFN is at bit 0 -- 54. */ 130 if (entry & PM_PRESENT) 131 return entry & 0x007fffffffffffffull; 132 return -1ul; 133 } 134 135 void clear_softdirty(void) 136 { 137 int ret; 138 const char *ctrl = "4"; 139 int fd = open("/proc/self/clear_refs", O_WRONLY); 140 141 if (fd < 0) 142 ksft_exit_fail_msg("opening clear_refs failed\n"); 143 ret = write(fd, ctrl, strlen(ctrl)); 144 close(fd); 145 if (ret != (signed int)strlen(ctrl)) 146 ksft_exit_fail_msg("writing clear_refs failed\n"); 147 } 148 149 bool check_for_pattern(FILE *fp, const char *pattern, char *buf, size_t len) 150 { 151 while (fgets(buf, len, fp)) { 152 if (!strncmp(buf, pattern, strlen(pattern))) 153 return true; 154 } 155 return false; 156 } 157 158 uint64_t read_pmd_pagesize(void) 159 { 160 int fd; 161 char buf[20]; 162 ssize_t num_read; 163 164 fd = open(PMD_SIZE_FILE_PATH, O_RDONLY); 165 if (fd == -1) 166 return 0; 167 168 num_read = read(fd, buf, 19); 169 if (num_read < 1) { 170 close(fd); 171 return 0; 172 } 173 buf[num_read] = '\0'; 174 close(fd); 175 176 return strtoul(buf, NULL, 10); 177 } 178 179 unsigned long rss_anon(void) 180 { 181 unsigned long rss_anon = 0; 182 FILE *fp; 183 char buffer[MAX_LINE_LENGTH]; 184 185 fp = fopen(STATUS_FILE_PATH, "r"); 186 if (!fp) 187 ksft_exit_fail_msg("%s: Failed to open file %s\n", __func__, STATUS_FILE_PATH); 188 189 if (!check_for_pattern(fp, "RssAnon:", buffer, sizeof(buffer))) 190 goto err_out; 191 192 if (sscanf(buffer, "RssAnon:%10lu kB", &rss_anon) != 1) 193 ksft_exit_fail_msg("Reading status error\n"); 194 195 err_out: 196 fclose(fp); 197 return rss_anon; 198 } 199 200 static int vaddr_pageflags_get(char *vaddr, int pagemap_fd, int kpageflags_fd, 201 uint64_t *flags) 202 { 203 unsigned long pfn; 204 205 pfn = pagemap_get_pfn(pagemap_fd, vaddr); 206 207 /* non-present PFN */ 208 if (pfn == -1UL) 209 return 1; 210 211 if (pageflags_get(pfn, kpageflags_fd, flags)) 212 return -1; 213 214 return 0; 215 } 216 217 /* 218 * gather_folio_orders - scan through [vaddr_start, len) and record 219 * folio orders 220 * 221 * @vaddr_start: start vaddr 222 * @len: range length 223 * @pagemap_fd: file descriptor to /proc/<pid>/pagemap 224 * @kpageflags_fd: file descriptor to /proc/kpageflags 225 * @orders: output folio order array 226 * @nr_orders: folio order array size 227 * 228 * gather_folio_orders() scan through [vaddr_start, len) and check 229 * all folios within the range and record their orders. All order-0 pages will 230 * be recorded. Non-present vaddr is skipped. 231 * 232 * Return: 0 - no error, -1 - unhandled cases 233 */ 234 int gather_folio_orders(char *vaddr_start, size_t len, 235 int pagemap_fd, int kpageflags_fd, int orders[], int nr_orders) 236 { 237 uint64_t page_flags = 0; 238 int cur_order = -1; 239 char *vaddr; 240 241 if (pagemap_fd == -1 || kpageflags_fd == -1) 242 return -1; 243 if (!orders) 244 return -1; 245 if (nr_orders <= 0) 246 return -1; 247 248 for (vaddr = vaddr_start; vaddr < vaddr_start + len;) { 249 char *next_folio_vaddr; 250 int status; 251 252 status = vaddr_pageflags_get(vaddr, pagemap_fd, kpageflags_fd, 253 &page_flags); 254 if (status < 0) 255 return -1; 256 257 /* skip non present vaddr */ 258 if (status == 1) { 259 vaddr += psize(); 260 continue; 261 } 262 263 /* all order-0 pages with possible false postive (non folio) */ 264 if (!(page_flags & (KPF_COMPOUND_HEAD | KPF_COMPOUND_TAIL))) { 265 orders[0]++; 266 vaddr += psize(); 267 continue; 268 } 269 270 /* skip non thp compound pages */ 271 if (!(page_flags & KPF_THP)) { 272 vaddr += psize(); 273 continue; 274 } 275 276 /* vpn points to part of a THP at this point */ 277 if (page_flags & KPF_COMPOUND_HEAD) 278 cur_order = 1; 279 else { 280 vaddr += psize(); 281 continue; 282 } 283 284 next_folio_vaddr = vaddr + (1UL << (cur_order + pshift())); 285 286 if (next_folio_vaddr >= vaddr_start + len) 287 break; 288 289 while ((status = vaddr_pageflags_get(next_folio_vaddr, 290 pagemap_fd, kpageflags_fd, 291 &page_flags)) >= 0) { 292 /* 293 * non present vaddr, next compound head page, or 294 * order-0 page 295 */ 296 if (status == 1 || 297 (page_flags & KPF_COMPOUND_HEAD) || 298 !(page_flags & (KPF_COMPOUND_HEAD | KPF_COMPOUND_TAIL))) { 299 if (cur_order < nr_orders) { 300 orders[cur_order]++; 301 cur_order = -1; 302 vaddr = next_folio_vaddr; 303 } 304 break; 305 } 306 307 cur_order++; 308 next_folio_vaddr = vaddr + (1UL << (cur_order + pshift())); 309 } 310 311 if (status < 0) 312 return status; 313 } 314 if (cur_order > 0 && cur_order < nr_orders) 315 orders[cur_order]++; 316 return 0; 317 } 318 319 char *__get_smap_entry(void *addr, const char *pattern, char *buf, size_t len) 320 { 321 int ret; 322 FILE *fp; 323 char *entry = NULL; 324 char addr_pattern[MAX_LINE_LENGTH]; 325 326 ret = snprintf(addr_pattern, MAX_LINE_LENGTH, "%08lx-", 327 (unsigned long) addr); 328 if (ret >= MAX_LINE_LENGTH) 329 ksft_exit_fail_msg("%s: Pattern is too long\n", __func__); 330 331 fp = fopen(SMAP_FILE_PATH, "r"); 332 if (!fp) 333 ksft_exit_fail_msg("%s: Failed to open file %s\n", __func__, SMAP_FILE_PATH); 334 335 if (!check_for_pattern(fp, addr_pattern, buf, len)) 336 goto err_out; 337 338 /* Fetch the pattern in the same block */ 339 if (!check_for_pattern(fp, pattern, buf, len)) 340 goto err_out; 341 342 /* Trim trailing newline */ 343 entry = strchr(buf, '\n'); 344 if (entry) 345 *entry = '\0'; 346 347 entry = buf + strlen(pattern); 348 349 err_out: 350 fclose(fp); 351 return entry; 352 } 353 354 static bool __check_pmd_huge(void *addr, char *pattern, int nr_hpages, 355 uint64_t hpage_size) 356 { 357 char buffer[MAX_LINE_LENGTH]; 358 uint64_t thp = -1; 359 char *entry; 360 361 entry = __get_smap_entry(addr, pattern, buffer, sizeof(buffer)); 362 if (!entry) 363 goto err_out; 364 365 if (sscanf(entry, "%9" SCNu64 " kB", &thp) != 1) 366 ksft_exit_fail_msg("Reading smap error\n"); 367 368 err_out: 369 return thp == (nr_hpages * (hpage_size >> 10)); 370 } 371 372 static bool check_large_folios(void *addr, size_t len, int nr_hpages, 373 uint64_t hpage_size) 374 { 375 int order = 0, pagesize = getpagesize(); 376 unsigned int nr_pages = hpage_size / pagesize; 377 int orders[MAX_NR_ORDERS], status; 378 int pagemap_fd, kpageflags_fd; 379 bool ret = false; 380 381 if (!nr_pages) 382 ksft_exit_fail_msg("invalid hugepage size\n"); 383 384 order = 31 - __builtin_clz(nr_pages); 385 if (!order || order >= MAX_NR_ORDERS) 386 ksft_exit_fail_msg("invalid order\n"); 387 388 memset(orders, 0, sizeof(int) * MAX_NR_ORDERS); 389 pagemap_fd = open(PAGEMAP_PATH, O_RDONLY); 390 if (pagemap_fd == -1) 391 ksft_exit_fail_msg("read pagemap fail\n"); 392 393 kpageflags_fd = open(KPAGEFLAGS_PATH, O_RDONLY); 394 if (kpageflags_fd == -1) { 395 close(pagemap_fd); 396 ksft_exit_fail_msg("read kpageflags fail\n"); 397 } 398 399 status = gather_folio_orders(addr, len, pagemap_fd, 400 kpageflags_fd, orders, MAX_NR_ORDERS); 401 if (status) 402 goto out; 403 404 if (orders[order] == nr_hpages) 405 ret = true; 406 407 out: 408 close(pagemap_fd); 409 close(kpageflags_fd); 410 return ret; 411 } 412 413 bool check_huge_anon(void *addr, size_t len, int nr_hpages, uint64_t hpage_size) 414 { 415 uint64_t pmd_pagesize = read_pmd_pagesize(); 416 417 if (!pmd_pagesize) 418 ksft_exit_fail_msg("reading PMD pagesize failed\n"); 419 420 if (hpage_size == pmd_pagesize) 421 return __check_pmd_huge(addr, "AnonHugePages: ", nr_hpages, hpage_size); 422 423 return check_large_folios(addr, len, nr_hpages, hpage_size); 424 } 425 426 bool check_huge_file(void *addr, size_t len, int nr_hpages, uint64_t hpage_size) 427 { 428 uint64_t pmd_pagesize = read_pmd_pagesize(); 429 430 if (!pmd_pagesize) 431 ksft_exit_fail_msg("reading PMD pagesize failed\n"); 432 433 if (hpage_size == pmd_pagesize) 434 return __check_pmd_huge(addr, "FilePmdMapped:", nr_hpages, hpage_size); 435 436 return check_large_folios(addr, len, nr_hpages, hpage_size); 437 } 438 439 bool check_huge_shmem(void *addr, size_t len, int nr_hpages, uint64_t hpage_size) 440 { 441 uint64_t pmd_pagesize = read_pmd_pagesize(); 442 443 if (!pmd_pagesize) 444 ksft_exit_fail_msg("reading PMD pagesize failed\n"); 445 446 if (hpage_size == pmd_pagesize) 447 return __check_pmd_huge(addr, "ShmemPmdMapped:", nr_hpages, hpage_size); 448 449 return check_large_folios(addr, len, nr_hpages, hpage_size); 450 } 451 452 int64_t allocate_transhuge(void *ptr, int pagemap_fd) 453 { 454 uint64_t ent[2]; 455 456 /* drop pmd */ 457 if (mmap(ptr, HPAGE_SIZE, PROT_READ | PROT_WRITE, 458 MAP_FIXED | MAP_ANONYMOUS | 459 MAP_NORESERVE | MAP_PRIVATE, -1, 0) != ptr) 460 ksft_exit_fail_msg("mmap transhuge\n"); 461 462 if (madvise(ptr, HPAGE_SIZE, MADV_HUGEPAGE)) 463 ksft_exit_fail_msg("MADV_HUGEPAGE\n"); 464 465 /* allocate transparent huge page */ 466 *(volatile void **)ptr = ptr; 467 468 if (pread(pagemap_fd, ent, sizeof(ent), 469 (uintptr_t)ptr >> (pshift() - 3)) != sizeof(ent)) 470 ksft_exit_fail_msg("read pagemap\n"); 471 472 if (PAGEMAP_PRESENT(ent[0]) && PAGEMAP_PRESENT(ent[1]) && 473 PAGEMAP_PFN(ent[0]) + 1 == PAGEMAP_PFN(ent[1]) && 474 !(PAGEMAP_PFN(ent[0]) & ((1 << (HPAGE_SHIFT - pshift())) - 1))) 475 return PAGEMAP_PFN(ent[0]); 476 477 return -1; 478 } 479 480 int pageflags_get(unsigned long pfn, int kpageflags_fd, uint64_t *flags) 481 { 482 size_t count; 483 484 count = pread(kpageflags_fd, flags, sizeof(*flags), 485 pfn * sizeof(*flags)); 486 487 if (count != sizeof(*flags)) 488 return -1; 489 490 return 0; 491 } 492 493 /* If `ioctls' non-NULL, the allowed ioctls will be returned into the var */ 494 int uffd_register_with_ioctls(int uffd, void *addr, uint64_t len, 495 bool miss, bool wp, bool minor, uint64_t *ioctls) 496 { 497 struct uffdio_register uffdio_register = { 0 }; 498 uint64_t mode = 0; 499 int ret = 0; 500 501 if (miss) 502 mode |= UFFDIO_REGISTER_MODE_MISSING; 503 if (wp) 504 mode |= UFFDIO_REGISTER_MODE_WP; 505 if (minor) 506 mode |= UFFDIO_REGISTER_MODE_MINOR; 507 508 uffdio_register.range.start = (unsigned long)addr; 509 uffdio_register.range.len = len; 510 uffdio_register.mode = mode; 511 512 if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register) == -1) 513 ret = -errno; 514 else if (ioctls) 515 *ioctls = uffdio_register.ioctls; 516 517 return ret; 518 } 519 520 int uffd_register(int uffd, void *addr, uint64_t len, 521 bool miss, bool wp, bool minor) 522 { 523 return uffd_register_with_ioctls(uffd, addr, len, 524 miss, wp, minor, NULL); 525 } 526 527 int uffd_unregister(int uffd, void *addr, uint64_t len) 528 { 529 struct uffdio_range range = { .start = (uintptr_t)addr, .len = len }; 530 int ret = 0; 531 532 if (ioctl(uffd, UFFDIO_UNREGISTER, &range) == -1) 533 ret = -errno; 534 535 return ret; 536 } 537 538 static bool check_vmflag(void *addr, const char *flag) 539 { 540 char buffer[MAX_LINE_LENGTH]; 541 const char *flags; 542 size_t flaglen; 543 544 flags = __get_smap_entry(addr, "VmFlags:", buffer, sizeof(buffer)); 545 if (!flags) 546 ksft_exit_fail_msg("%s: No VmFlags for %p\n", __func__, addr); 547 548 while (true) { 549 flags += strspn(flags, " "); 550 551 flaglen = strcspn(flags, " "); 552 if (!flaglen) 553 return false; 554 555 if (flaglen == strlen(flag) && !memcmp(flags, flag, flaglen)) 556 return true; 557 558 flags += flaglen; 559 } 560 } 561 562 bool check_vmflag_io(void *addr) 563 { 564 return check_vmflag(addr, "io"); 565 } 566 567 bool check_vmflag_pfnmap(void *addr) 568 { 569 return check_vmflag(addr, "pf"); 570 } 571 572 bool check_vmflag_guard(void *addr) 573 { 574 return check_vmflag(addr, "gu"); 575 } 576 577 bool softdirty_supported(void) 578 { 579 char *addr; 580 bool supported = false; 581 const size_t pagesize = getpagesize(); 582 583 /* New mappings are expected to be marked with VM_SOFTDIRTY (sd). */ 584 addr = mmap(0, pagesize, PROT_READ | PROT_WRITE, 585 MAP_ANONYMOUS | MAP_PRIVATE, 0, 0); 586 if (addr == MAP_FAILED) 587 ksft_exit_fail_msg("mmap failed\n"); 588 589 supported = check_vmflag(addr, "sd"); 590 munmap(addr, pagesize); 591 return supported; 592 } 593 594 /* 595 * Open an fd at /proc/$pid/maps and configure procmap_out ready for 596 * PROCMAP_QUERY query. Returns 0 on success, or an error code otherwise. 597 */ 598 int open_procmap(pid_t pid, struct procmap_fd *procmap_out) 599 { 600 char path[256]; 601 int ret = 0; 602 603 memset(procmap_out, '\0', sizeof(*procmap_out)); 604 sprintf(path, "/proc/%d/maps", pid); 605 procmap_out->query.size = sizeof(procmap_out->query); 606 procmap_out->fd = open(path, O_RDONLY); 607 if (procmap_out->fd < 0) 608 ret = -errno; 609 610 return ret; 611 } 612 613 /* Perform PROCMAP_QUERY. Returns 0 on success, or an error code otherwise. */ 614 int query_procmap(struct procmap_fd *procmap) 615 { 616 int ret = 0; 617 618 if (ioctl(procmap->fd, PROCMAP_QUERY, &procmap->query) == -1) 619 ret = -errno; 620 621 return ret; 622 } 623 624 /* 625 * Try to find the VMA at specified address, returns true if found, false if not 626 * found, and the test is failed if any other error occurs. 627 * 628 * On success, procmap->query is populated with the results. 629 */ 630 bool find_vma_procmap(struct procmap_fd *procmap, void *address) 631 { 632 int err; 633 634 procmap->query.query_flags = 0; 635 procmap->query.query_addr = (unsigned long)address; 636 err = query_procmap(procmap); 637 if (!err) 638 return true; 639 640 if (err != -ENOENT) 641 ksft_exit_fail_msg("%s: Error %d on ioctl(PROCMAP_QUERY)\n", 642 __func__, err); 643 return false; 644 } 645 646 /* 647 * Close fd used by PROCMAP_QUERY mechanism. Returns 0 on success, or an error 648 * code otherwise. 649 */ 650 int close_procmap(struct procmap_fd *procmap) 651 { 652 return close(procmap->fd); 653 } 654 655 int write_sysfs(const char *file_path, unsigned long val) 656 { 657 FILE *f = fopen(file_path, "w"); 658 659 if (!f) { 660 fprintf(stderr, "f %s\n", file_path); 661 perror("fopen"); 662 return 1; 663 } 664 if (fprintf(f, "%lu", val) < 0) { 665 perror("fprintf"); 666 fclose(f); 667 return 1; 668 } 669 fclose(f); 670 671 return 0; 672 } 673 674 int read_sysfs(const char *file_path, unsigned long *val) 675 { 676 FILE *f = fopen(file_path, "r"); 677 678 if (!f) { 679 fprintf(stderr, "f %s\n", file_path); 680 perror("fopen"); 681 return 1; 682 } 683 if (fscanf(f, "%lu", val) != 1) { 684 perror("fscanf"); 685 fclose(f); 686 return 1; 687 } 688 fclose(f); 689 690 return 0; 691 } 692 693 void *sys_mremap(void *old_address, unsigned long old_size, 694 unsigned long new_size, int flags, void *new_address) 695 { 696 return (void *)syscall(__NR_mremap, (unsigned long)old_address, 697 old_size, new_size, flags, 698 (unsigned long)new_address); 699 } 700 701 bool detect_huge_zeropage(void) 702 { 703 int fd = open("/sys/kernel/mm/transparent_hugepage/use_zero_page", 704 O_RDONLY); 705 bool enabled = 0; 706 char buf[15]; 707 int ret; 708 709 if (fd < 0) 710 return 0; 711 712 ret = pread(fd, buf, sizeof(buf), 0); 713 if (ret > 0 && ret < sizeof(buf)) { 714 buf[ret] = 0; 715 716 if (strtoul(buf, NULL, 10) == 1) 717 enabled = 1; 718 } 719 720 close(fd); 721 return enabled; 722 } 723 724 long ksm_get_self_zero_pages(void) 725 { 726 int proc_self_ksm_stat_fd; 727 char buf[200]; 728 char *substr_ksm_zero; 729 size_t value_pos; 730 ssize_t read_size; 731 732 proc_self_ksm_stat_fd = open("/proc/self/ksm_stat", O_RDONLY); 733 if (proc_self_ksm_stat_fd < 0) 734 return -errno; 735 736 read_size = pread(proc_self_ksm_stat_fd, buf, sizeof(buf) - 1, 0); 737 close(proc_self_ksm_stat_fd); 738 if (read_size < 0) 739 return -errno; 740 741 buf[read_size] = 0; 742 743 substr_ksm_zero = strstr(buf, "ksm_zero_pages"); 744 if (!substr_ksm_zero) 745 return 0; 746 747 value_pos = strcspn(substr_ksm_zero, "0123456789"); 748 return strtol(substr_ksm_zero + value_pos, NULL, 10); 749 } 750 751 long ksm_get_self_merging_pages(void) 752 { 753 int proc_self_ksm_merging_pages_fd; 754 char buf[10]; 755 ssize_t ret; 756 757 proc_self_ksm_merging_pages_fd = open("/proc/self/ksm_merging_pages", 758 O_RDONLY); 759 if (proc_self_ksm_merging_pages_fd < 0) 760 return -errno; 761 762 ret = pread(proc_self_ksm_merging_pages_fd, buf, sizeof(buf) - 1, 0); 763 close(proc_self_ksm_merging_pages_fd); 764 if (ret <= 0) 765 return -errno; 766 buf[ret] = 0; 767 768 return strtol(buf, NULL, 10); 769 } 770 771 long ksm_get_full_scans(void) 772 { 773 int ksm_full_scans_fd; 774 char buf[10]; 775 ssize_t ret; 776 777 ksm_full_scans_fd = open("/sys/kernel/mm/ksm/full_scans", O_RDONLY); 778 if (ksm_full_scans_fd < 0) 779 return -errno; 780 781 ret = pread(ksm_full_scans_fd, buf, sizeof(buf) - 1, 0); 782 close(ksm_full_scans_fd); 783 if (ret <= 0) 784 return -errno; 785 buf[ret] = 0; 786 787 return strtol(buf, NULL, 10); 788 } 789 790 int ksm_use_zero_pages(void) 791 { 792 int ksm_use_zero_pages_fd; 793 ssize_t ret; 794 795 ksm_use_zero_pages_fd = open("/sys/kernel/mm/ksm/use_zero_pages", O_RDWR); 796 if (ksm_use_zero_pages_fd < 0) 797 return -errno; 798 799 ret = write(ksm_use_zero_pages_fd, "1", 1); 800 close(ksm_use_zero_pages_fd); 801 return ret == 1 ? 0 : -errno; 802 } 803 804 int ksm_start(void) 805 { 806 int ksm_fd; 807 ssize_t ret; 808 long start_scans, end_scans; 809 810 ksm_fd = open("/sys/kernel/mm/ksm/run", O_RDWR); 811 if (ksm_fd < 0) 812 return -errno; 813 814 /* Wait for two full scans such that any possible merging happened. */ 815 start_scans = ksm_get_full_scans(); 816 if (start_scans < 0) { 817 close(ksm_fd); 818 return start_scans; 819 } 820 ret = write(ksm_fd, "1", 1); 821 close(ksm_fd); 822 if (ret != 1) 823 return -errno; 824 do { 825 end_scans = ksm_get_full_scans(); 826 if (end_scans < 0) 827 return end_scans; 828 } while (end_scans < start_scans + 2); 829 830 return 0; 831 } 832 833 int ksm_stop(void) 834 { 835 int ksm_fd; 836 ssize_t ret; 837 838 ksm_fd = open("/sys/kernel/mm/ksm/run", O_RDWR); 839 if (ksm_fd < 0) 840 return -errno; 841 842 ret = write(ksm_fd, "2", 1); 843 close(ksm_fd); 844 return ret == 1 ? 0 : -errno; 845 } 846 847 int get_hardware_corrupted_size(unsigned long *val) 848 { 849 unsigned long size; 850 char *line = NULL; 851 size_t linelen = 0; 852 FILE *f = fopen("/proc/meminfo", "r"); 853 int ret = -1; 854 855 if (!f) 856 return ret; 857 858 while (getline(&line, &linelen, f) > 0) { 859 if (sscanf(line, "HardwareCorrupted: %12lu kB", &size) == 1) { 860 *val = size; 861 ret = 0; 862 break; 863 } 864 } 865 866 free(line); 867 fclose(f); 868 return ret; 869 } 870 871 int unpoison_memory(unsigned long pfn) 872 { 873 int unpoison_fd, len; 874 char buf[32]; 875 ssize_t ret; 876 877 unpoison_fd = open("/sys/kernel/debug/hwpoison/unpoison-pfn", O_WRONLY); 878 if (unpoison_fd < 0) 879 return -errno; 880 881 len = sprintf(buf, "0x%lx\n", pfn); 882 ret = write(unpoison_fd, buf, len); 883 close(unpoison_fd); 884 885 return ret > 0 ? 0 : -errno; 886 } 887 888 int read_file(const char *path, char *buf, size_t buflen) 889 { 890 int fd; 891 ssize_t numread; 892 893 fd = open(path, O_RDONLY); 894 if (fd == -1) 895 return 0; 896 897 numread = read(fd, buf, buflen - 1); 898 if (numread < 1) { 899 close(fd); 900 return 0; 901 } 902 903 buf[numread] = '\0'; 904 close(fd); 905 906 return (unsigned int) numread; 907 } 908 909 static void __write_file(const char *path, const char *buf, size_t buflen, bool ignore_einval) 910 { 911 int fd, saved_errno; 912 ssize_t numwritten; 913 914 if (buflen < 2) 915 ksft_exit_fail_msg("Incorrect buffer len: %zu\n", buflen); 916 917 fd = open(path, O_WRONLY); 918 if (fd == -1) 919 ksft_exit_fail_msg("%s open failed: %s\n", path, strerror(errno)); 920 921 numwritten = write(fd, buf, buflen - 1); 922 saved_errno = errno; 923 close(fd); 924 errno = saved_errno; 925 if (numwritten < 0) { 926 if (ignore_einval && errno == EINVAL) 927 return; 928 ksft_exit_fail_msg("%s write(%.*s) failed: %s\n", path, (int)(buflen - 1), 929 buf, strerror(errno)); 930 } 931 if (numwritten != buflen - 1) 932 ksft_exit_fail_msg("%s write(%.*s) is truncated, expected %zu bytes, got %zd bytes\n", 933 path, (int)(buflen - 1), buf, buflen - 1, numwritten); 934 } 935 936 void write_file(const char *path, const char *buf, size_t buflen) 937 { 938 __write_file(path, buf, buflen, /* ignore_einval = */ false); 939 } 940 941 unsigned long read_num(const char *path) 942 { 943 char buf[21]; 944 945 if (!read_file(path, buf, sizeof(buf))) 946 ksft_exit_fail_perror("read_file()"); 947 948 return strtoul(buf, NULL, 10); 949 } 950 951 static void __write_num(const char *path, unsigned long num, bool ignore_einval) 952 { 953 char buf[21]; 954 955 sprintf(buf, "%lu", num); 956 __write_file(path, buf, strlen(buf) + 1, ignore_einval); 957 } 958 959 void write_num(const char *path, unsigned long num) 960 { 961 return __write_num(path, num, /* ignore_einval = */ false); 962 } 963 964 void write_num_ignore_einval(const char *path, unsigned long num) 965 { 966 return __write_num(path, num, /* ignore_einval = */ true); 967 } 968 969 static unsigned long shmall, shmmax; 970 971 void __shm_limits_restore(void) 972 { 973 if (shmmax) 974 write_num("/proc/sys/kernel/shmmax", shmmax); 975 if (shmall) 976 write_num("/proc/sys/kernel/shmall", shmall); 977 } 978 979 void shm_limits_prepare(unsigned long length) 980 { 981 unsigned long nr = length / psize(); 982 unsigned long val; 983 984 val = read_num("/proc/sys/kernel/shmmax"); 985 if (val < length) { 986 write_num("/proc/sys/kernel/shmmax", length); 987 shmmax = val; 988 } 989 990 val = read_num("/proc/sys/kernel/shmall"); 991 if (val < nr) { 992 write_num("/proc/sys/kernel/shmall", nr); 993 shmall = val; 994 } 995 } 996