1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * page-types: Tool for querying page flags 4 * 5 * Copyright (C) 2009 Intel corporation 6 * 7 * Authors: Wu Fengguang <fengguang.wu@intel.com> 8 */ 9 10 #define _FILE_OFFSET_BITS 64 11 #define _GNU_SOURCE 12 #include <stdio.h> 13 #include <stdlib.h> 14 #include <unistd.h> 15 #include <stdint.h> 16 #include <stdarg.h> 17 #include <string.h> 18 #include <getopt.h> 19 #include <limits.h> 20 #include <assert.h> 21 #include <ftw.h> 22 #include <time.h> 23 #include <setjmp.h> 24 #include <signal.h> 25 #include <inttypes.h> 26 #include <sys/types.h> 27 #include <sys/errno.h> 28 #include <sys/fcntl.h> 29 #include <sys/mount.h> 30 #include <sys/statfs.h> 31 #include <sys/mman.h> 32 #include "../../include/uapi/linux/magic.h" 33 #include "../../include/uapi/linux/kernel-page-flags.h" 34 #include <api/fs/fs.h> 35 36 #ifndef MAX_PATH 37 # define MAX_PATH 256 38 #endif 39 40 #ifndef STR 41 # define _STR(x) #x 42 # define STR(x) _STR(x) 43 #endif 44 45 /* 46 * pagemap kernel ABI bits 47 */ 48 49 #define PM_ENTRY_BYTES 8 50 #define PM_PFRAME_BITS 55 51 #define PM_PFRAME_MASK ((1LL << PM_PFRAME_BITS) - 1) 52 #define PM_PFRAME(x) ((x) & PM_PFRAME_MASK) 53 #define MAX_SWAPFILES_SHIFT 5 54 #define PM_SWAP_OFFSET(x) (((x) & PM_PFRAME_MASK) >> MAX_SWAPFILES_SHIFT) 55 #define PM_SOFT_DIRTY (1ULL << 55) 56 #define PM_MMAP_EXCLUSIVE (1ULL << 56) 57 #define PM_FILE (1ULL << 61) 58 #define PM_SWAP (1ULL << 62) 59 #define PM_PRESENT (1ULL << 63) 60 61 /* 62 * kernel page flags 63 */ 64 65 #define KPF_BYTES 8 66 #define PROC_KPAGEFLAGS "/proc/kpageflags" 67 #define PROC_KPAGECOUNT "/proc/kpagecount" 68 #define PROC_KPAGECGROUP "/proc/kpagecgroup" 69 70 #define SYS_KERNEL_MM_PAGE_IDLE "/sys/kernel/mm/page_idle/bitmap" 71 72 /* [32-] kernel hacking assistances */ 73 #define KPF_RESERVED 32 74 #define KPF_MLOCKED 33 75 #define KPF_OWNER_2 34 76 #define KPF_PRIVATE 35 77 #define KPF_PRIVATE_2 36 78 #define KPF_OWNER_PRIVATE 37 79 #define KPF_ARCH 38 80 #define KPF_UNCACHED 39 /* unused */ 81 #define KPF_SOFTDIRTY 40 82 #define KPF_ARCH_2 41 83 84 /* [47-] take some arbitrary free slots for expanding overloaded flags 85 * not part of kernel API 86 */ 87 #define KPF_ANON_EXCLUSIVE 47 88 #define KPF_READAHEAD 48 89 #define KPF_SLUB_FROZEN 50 90 #define KPF_SLUB_DEBUG 51 91 #define KPF_FILE 61 92 #define KPF_SWAP 62 93 #define KPF_MMAP_EXCLUSIVE 63 94 95 #define KPF_ALL_BITS ((uint64_t)~0ULL) 96 #define KPF_HACKERS_BITS (0xffffULL << 32) 97 #define KPF_OVERLOADED_BITS (0xffffULL << 48) 98 #define BIT(name) (1ULL << KPF_##name) 99 #define BITS_COMPOUND (BIT(COMPOUND_HEAD) | BIT(COMPOUND_TAIL)) 100 101 static const char * const page_flag_names[] = { 102 [KPF_LOCKED] = "L:locked", 103 [KPF_ERROR] = "E:error", 104 [KPF_REFERENCED] = "R:referenced", 105 [KPF_UPTODATE] = "U:uptodate", 106 [KPF_DIRTY] = "D:dirty", 107 [KPF_LRU] = "l:lru", 108 [KPF_ACTIVE] = "A:active", 109 [KPF_SLAB] = "S:slab", 110 [KPF_WRITEBACK] = "W:writeback", 111 [KPF_RECLAIM] = "I:reclaim", 112 [KPF_BUDDY] = "B:buddy", 113 114 [KPF_MMAP] = "M:mmap", 115 [KPF_ANON] = "a:anonymous", 116 [KPF_SWAPCACHE] = "s:swapcache", 117 [KPF_SWAPBACKED] = "b:swapbacked", 118 [KPF_COMPOUND_HEAD] = "H:compound_head", 119 [KPF_COMPOUND_TAIL] = "T:compound_tail", 120 [KPF_HUGE] = "G:huge", 121 [KPF_UNEVICTABLE] = "u:unevictable", 122 [KPF_HWPOISON] = "X:hwpoison", 123 [KPF_NOPAGE] = "n:nopage", 124 [KPF_KSM] = "x:ksm", 125 [KPF_THP] = "t:thp", 126 [KPF_OFFLINE] = "o:offline", 127 [KPF_PGTABLE] = "g:pgtable", 128 [KPF_ZERO_PAGE] = "z:zero_page", 129 [KPF_IDLE] = "i:idle_page", 130 131 [KPF_RESERVED] = "r:reserved", 132 [KPF_MLOCKED] = "m:mlocked", 133 [KPF_OWNER_2] = "d:owner_2", 134 [KPF_PRIVATE] = "P:private", 135 [KPF_PRIVATE_2] = "p:private_2", 136 [KPF_OWNER_PRIVATE] = "O:owner_private", 137 [KPF_ARCH] = "h:arch", 138 [KPF_SOFTDIRTY] = "f:softdirty", 139 [KPF_ARCH_2] = "H:arch_2", 140 141 [KPF_ANON_EXCLUSIVE] = "d:anon_exclusive", 142 [KPF_READAHEAD] = "I:readahead", 143 [KPF_SLUB_FROZEN] = "A:slub_frozen", 144 [KPF_SLUB_DEBUG] = "E:slub_debug", 145 146 [KPF_FILE] = "F:file", 147 [KPF_SWAP] = "w:swap", 148 [KPF_MMAP_EXCLUSIVE] = "1:mmap_exclusive", 149 }; 150 151 152 /* 153 * data structures 154 */ 155 156 static int opt_raw; /* for kernel developers */ 157 static int opt_list; /* list pages (in ranges) */ 158 static int opt_mark_idle; /* set accessed bit */ 159 static int opt_no_summary; /* don't show summary */ 160 static pid_t opt_pid; /* process to walk */ 161 const char *opt_file; /* file or directory path */ 162 static uint64_t opt_cgroup; /* cgroup inode */ 163 static int opt_list_cgroup;/* list page cgroup */ 164 static int opt_list_mapcnt;/* list page map count */ 165 static const char *opt_kpageflags;/* kpageflags file to parse */ 166 167 #define MAX_ADDR_RANGES 1024 168 static int nr_addr_ranges; 169 static unsigned long opt_offset[MAX_ADDR_RANGES]; 170 static unsigned long opt_size[MAX_ADDR_RANGES]; 171 172 #define MAX_VMAS 10240 173 static int nr_vmas; 174 static unsigned long pg_start[MAX_VMAS]; 175 static unsigned long pg_end[MAX_VMAS]; 176 177 #define MAX_BIT_FILTERS 64 178 static int nr_bit_filters; 179 static uint64_t opt_mask[MAX_BIT_FILTERS]; 180 static uint64_t opt_bits[MAX_BIT_FILTERS]; 181 182 static int page_size; 183 184 static int pagemap_fd; 185 static int kpageflags_fd; 186 static int kpagecount_fd = -1; 187 static int kpagecgroup_fd = -1; 188 static int page_idle_fd = -1; 189 190 static int opt_hwpoison; 191 static int opt_unpoison; 192 193 static const char *hwpoison_debug_fs; 194 static int hwpoison_inject_fd; 195 static int hwpoison_forget_fd; 196 197 #define HASH_SHIFT 13 198 #define HASH_SIZE (1 << HASH_SHIFT) 199 #define HASH_MASK (HASH_SIZE - 1) 200 #define HASH_KEY(flags) (flags & HASH_MASK) 201 202 static unsigned long total_pages; 203 static unsigned long nr_pages[HASH_SIZE]; 204 static uint64_t page_flags[HASH_SIZE]; 205 206 207 /* 208 * helper functions 209 */ 210 211 #define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) 212 213 #define min_t(type, x, y) ({ \ 214 type __min1 = (x); \ 215 type __min2 = (y); \ 216 __min1 < __min2 ? __min1 : __min2; }) 217 218 #define max_t(type, x, y) ({ \ 219 type __max1 = (x); \ 220 type __max2 = (y); \ 221 __max1 > __max2 ? __max1 : __max2; }) 222 223 static unsigned long pages2mb(unsigned long pages) 224 { 225 return (pages * page_size) >> 20; 226 } 227 228 static void fatal(const char *x, ...) 229 { 230 va_list ap; 231 232 va_start(ap, x); 233 vfprintf(stderr, x, ap); 234 va_end(ap); 235 exit(EXIT_FAILURE); 236 } 237 238 static int checked_open(const char *pathname, int flags) 239 { 240 int fd = open(pathname, flags); 241 242 if (fd < 0) { 243 perror(pathname); 244 exit(EXIT_FAILURE); 245 } 246 247 return fd; 248 } 249 250 /* 251 * pagemap/kpageflags routines 252 */ 253 254 static unsigned long do_u64_read(int fd, const char *name, 255 uint64_t *buf, 256 unsigned long index, 257 unsigned long count) 258 { 259 long bytes; 260 261 if (index > ULONG_MAX / 8) 262 fatal("index overflow: %lu\n", index); 263 264 bytes = pread(fd, buf, count * 8, (off_t)index * 8); 265 if (bytes < 0) { 266 perror(name); 267 exit(EXIT_FAILURE); 268 } 269 if (bytes % 8) 270 fatal("partial read: %lu bytes\n", bytes); 271 272 return bytes / 8; 273 } 274 275 static unsigned long kpageflags_read(uint64_t *buf, 276 unsigned long index, 277 unsigned long pages) 278 { 279 return do_u64_read(kpageflags_fd, opt_kpageflags, buf, index, pages); 280 } 281 282 static unsigned long kpagecgroup_read(uint64_t *buf, 283 unsigned long index, 284 unsigned long pages) 285 { 286 if (kpagecgroup_fd < 0) 287 return pages; 288 289 return do_u64_read(kpagecgroup_fd, opt_kpageflags, buf, index, pages); 290 } 291 292 static unsigned long kpagecount_read(uint64_t *buf, 293 unsigned long index, 294 unsigned long pages) 295 { 296 return kpagecount_fd < 0 ? pages : 297 do_u64_read(kpagecount_fd, PROC_KPAGECOUNT, 298 buf, index, pages); 299 } 300 301 static unsigned long pagemap_read(uint64_t *buf, 302 unsigned long index, 303 unsigned long pages) 304 { 305 return do_u64_read(pagemap_fd, "/proc/pid/pagemap", buf, index, pages); 306 } 307 308 static unsigned long pagemap_pfn(uint64_t val) 309 { 310 unsigned long pfn; 311 312 if (val & PM_PRESENT) 313 pfn = PM_PFRAME(val); 314 else 315 pfn = 0; 316 317 return pfn; 318 } 319 320 static unsigned long pagemap_swap_offset(uint64_t val) 321 { 322 return val & PM_SWAP ? PM_SWAP_OFFSET(val) : 0; 323 } 324 325 /* 326 * page flag names 327 */ 328 329 static char *page_flag_name(uint64_t flags) 330 { 331 static char buf[65]; 332 int present; 333 size_t i, j; 334 335 for (i = 0, j = 0; i < ARRAY_SIZE(page_flag_names); i++) { 336 present = (flags >> i) & 1; 337 if (!page_flag_names[i]) { 338 if (present) 339 fatal("unknown flag bit %d\n", i); 340 continue; 341 } 342 buf[j++] = present ? page_flag_names[i][0] : '_'; 343 } 344 345 return buf; 346 } 347 348 static char *page_flag_longname(uint64_t flags) 349 { 350 static char buf[1024]; 351 size_t i, n; 352 353 for (i = 0, n = 0; i < ARRAY_SIZE(page_flag_names); i++) { 354 if (!page_flag_names[i]) 355 continue; 356 if ((flags >> i) & 1) 357 n += snprintf(buf + n, sizeof(buf) - n, "%s,", 358 page_flag_names[i] + 2); 359 } 360 if (n) 361 n--; 362 buf[n] = '\0'; 363 364 return buf; 365 } 366 367 368 /* 369 * page list and summary 370 */ 371 372 static void show_page_range(unsigned long voffset, unsigned long offset, 373 unsigned long size, uint64_t flags, 374 uint64_t cgroup, uint64_t mapcnt) 375 { 376 static uint64_t flags0; 377 static uint64_t cgroup0; 378 static uint64_t mapcnt0; 379 static unsigned long voff; 380 static unsigned long index; 381 static unsigned long count; 382 383 if (flags == flags0 && cgroup == cgroup0 && mapcnt == mapcnt0 && 384 offset == index + count && size && voffset == voff + count) { 385 count += size; 386 return; 387 } 388 389 if (count) { 390 if (opt_pid) 391 printf("%lx\t", voff); 392 if (opt_file) 393 printf("%lx\t", voff); 394 if (opt_list_cgroup) 395 printf("@%" PRIu64 "\t", cgroup0); 396 if (opt_list_mapcnt) 397 printf("%" PRIu64 "\t", mapcnt0); 398 printf("%lx\t%lx\t%s\n", 399 index, count, page_flag_name(flags0)); 400 } 401 402 flags0 = flags; 403 cgroup0 = cgroup; 404 mapcnt0 = mapcnt; 405 index = offset; 406 voff = voffset; 407 count = size; 408 } 409 410 static void flush_page_range(void) 411 { 412 show_page_range(0, 0, 0, 0, 0, 0); 413 } 414 415 static void show_page(unsigned long voffset, unsigned long offset, 416 uint64_t flags, uint64_t cgroup, uint64_t mapcnt) 417 { 418 if (opt_pid) 419 printf("%lx\t", voffset); 420 if (opt_file) 421 printf("%lx\t", voffset); 422 if (opt_list_cgroup) 423 printf("@%" PRIu64 "\t", cgroup); 424 if (opt_list_mapcnt) 425 printf("%" PRIu64 "\t", mapcnt); 426 427 printf("%lx\t%s\n", offset, page_flag_name(flags)); 428 } 429 430 static void show_summary(void) 431 { 432 size_t i; 433 434 printf(" flags\tpage-count MB" 435 " symbolic-flags\t\t\tlong-symbolic-flags\n"); 436 437 for (i = 0; i < ARRAY_SIZE(nr_pages); i++) { 438 if (nr_pages[i]) 439 printf("0x%016llx\t%10lu %8lu %s\t%s\n", 440 (unsigned long long)page_flags[i], 441 nr_pages[i], 442 pages2mb(nr_pages[i]), 443 page_flag_name(page_flags[i]), 444 page_flag_longname(page_flags[i])); 445 } 446 447 printf(" total\t%10lu %8lu\n", 448 total_pages, pages2mb(total_pages)); 449 } 450 451 452 /* 453 * page flag filters 454 */ 455 456 static int bit_mask_ok(uint64_t flags) 457 { 458 int i; 459 460 for (i = 0; i < nr_bit_filters; i++) { 461 if (opt_bits[i] == KPF_ALL_BITS) { 462 if ((flags & opt_mask[i]) == 0) 463 return 0; 464 } else { 465 if ((flags & opt_mask[i]) != opt_bits[i]) 466 return 0; 467 } 468 } 469 470 return 1; 471 } 472 473 static uint64_t expand_overloaded_flags(uint64_t flags, uint64_t pme) 474 { 475 /* Anonymous pages use PG_owner_2 for anon_exclusive */ 476 if ((flags & BIT(ANON)) && (flags & BIT(OWNER_2))) 477 flags ^= BIT(OWNER_2) | BIT(ANON_EXCLUSIVE); 478 479 /* SLUB overloads several page flags */ 480 if (flags & BIT(SLAB)) { 481 if (flags & BIT(ACTIVE)) 482 flags ^= BIT(ACTIVE) | BIT(SLUB_FROZEN); 483 if (flags & BIT(ERROR)) 484 flags ^= BIT(ERROR) | BIT(SLUB_DEBUG); 485 } 486 487 /* PG_reclaim is overloaded as PG_readahead in the read path */ 488 if ((flags & (BIT(RECLAIM) | BIT(WRITEBACK))) == BIT(RECLAIM)) 489 flags ^= BIT(RECLAIM) | BIT(READAHEAD); 490 491 if (pme & PM_SOFT_DIRTY) 492 flags |= BIT(SOFTDIRTY); 493 if (pme & PM_FILE) 494 flags |= BIT(FILE); 495 if (pme & PM_SWAP) 496 flags |= BIT(SWAP); 497 if (pme & PM_MMAP_EXCLUSIVE) 498 flags |= BIT(MMAP_EXCLUSIVE); 499 500 return flags; 501 } 502 503 static uint64_t well_known_flags(uint64_t flags) 504 { 505 /* hide flags intended only for kernel hacker */ 506 flags &= ~KPF_HACKERS_BITS; 507 508 /* hide non-hugeTLB compound pages */ 509 if ((flags & BITS_COMPOUND) && !(flags & BIT(HUGE))) 510 flags &= ~BITS_COMPOUND; 511 512 return flags; 513 } 514 515 static uint64_t kpageflags_flags(uint64_t flags, uint64_t pme) 516 { 517 if (opt_raw) 518 flags = expand_overloaded_flags(flags, pme); 519 else 520 flags = well_known_flags(flags); 521 522 return flags; 523 } 524 525 /* 526 * page actions 527 */ 528 529 static void prepare_hwpoison_fd(void) 530 { 531 char buf[MAX_PATH + 1]; 532 533 hwpoison_debug_fs = debugfs__mount(); 534 if (!hwpoison_debug_fs) { 535 perror("mount debugfs"); 536 exit(EXIT_FAILURE); 537 } 538 539 if (opt_hwpoison && !hwpoison_inject_fd) { 540 snprintf(buf, MAX_PATH, "%s/hwpoison/corrupt-pfn", 541 hwpoison_debug_fs); 542 hwpoison_inject_fd = checked_open(buf, O_WRONLY); 543 } 544 545 if (opt_unpoison && !hwpoison_forget_fd) { 546 snprintf(buf, MAX_PATH, "%s/hwpoison/unpoison-pfn", 547 hwpoison_debug_fs); 548 hwpoison_forget_fd = checked_open(buf, O_WRONLY); 549 } 550 } 551 552 static int hwpoison_page(unsigned long offset) 553 { 554 char buf[100]; 555 int len; 556 557 len = sprintf(buf, "0x%lx\n", offset); 558 len = write(hwpoison_inject_fd, buf, len); 559 if (len < 0) { 560 perror("hwpoison inject"); 561 return len; 562 } 563 return 0; 564 } 565 566 static int unpoison_page(unsigned long offset) 567 { 568 char buf[100]; 569 int len; 570 571 len = sprintf(buf, "0x%lx\n", offset); 572 len = write(hwpoison_forget_fd, buf, len); 573 if (len < 0) { 574 perror("hwpoison forget"); 575 return len; 576 } 577 return 0; 578 } 579 580 static int mark_page_idle(unsigned long offset) 581 { 582 static unsigned long off; 583 static uint64_t buf; 584 int len; 585 586 if ((offset / 64 == off / 64) || buf == 0) { 587 buf |= 1UL << (offset % 64); 588 off = offset; 589 return 0; 590 } 591 592 len = pwrite(page_idle_fd, &buf, 8, 8 * (off / 64)); 593 if (len < 0) { 594 perror("mark page idle"); 595 return len; 596 } 597 598 buf = 1UL << (offset % 64); 599 off = offset; 600 601 return 0; 602 } 603 604 /* 605 * page frame walker 606 */ 607 608 static size_t hash_slot(uint64_t flags) 609 { 610 size_t k = HASH_KEY(flags); 611 size_t i; 612 613 /* Explicitly reserve slot 0 for flags 0: the following logic 614 * cannot distinguish an unoccupied slot from slot (flags==0). 615 */ 616 if (flags == 0) 617 return 0; 618 619 /* search through the remaining (HASH_SIZE-1) slots */ 620 for (i = 1; i < ARRAY_SIZE(page_flags); i++, k++) { 621 if (!k || k >= ARRAY_SIZE(page_flags)) 622 k = 1; 623 if (page_flags[k] == 0) { 624 page_flags[k] = flags; 625 return k; 626 } 627 if (page_flags[k] == flags) 628 return k; 629 } 630 631 fatal("hash table full: bump up HASH_SHIFT?\n"); 632 exit(EXIT_FAILURE); 633 } 634 635 static void add_page(unsigned long voffset, unsigned long offset, 636 uint64_t flags, uint64_t cgroup, uint64_t mapcnt, 637 uint64_t pme) 638 { 639 flags = kpageflags_flags(flags, pme); 640 641 if (!bit_mask_ok(flags)) 642 return; 643 644 if (opt_cgroup && cgroup != (uint64_t)opt_cgroup) 645 return; 646 647 if (opt_hwpoison) 648 hwpoison_page(offset); 649 if (opt_unpoison) 650 unpoison_page(offset); 651 652 if (opt_mark_idle) 653 mark_page_idle(offset); 654 655 if (opt_list == 1) 656 show_page_range(voffset, offset, 1, flags, cgroup, mapcnt); 657 else if (opt_list == 2) 658 show_page(voffset, offset, flags, cgroup, mapcnt); 659 660 nr_pages[hash_slot(flags)]++; 661 total_pages++; 662 } 663 664 #define KPAGEFLAGS_BATCH (64 << 10) /* 64k pages */ 665 static void walk_pfn(unsigned long voffset, 666 unsigned long index, 667 unsigned long count, 668 uint64_t pme) 669 { 670 uint64_t buf[KPAGEFLAGS_BATCH]; 671 uint64_t cgi[KPAGEFLAGS_BATCH]; 672 uint64_t cnt[KPAGEFLAGS_BATCH]; 673 unsigned long batch; 674 unsigned long pages; 675 unsigned long i; 676 677 /* 678 * kpagecgroup_read() reads only if kpagecgroup were opened, but 679 * /proc/kpagecgroup might even not exist, so it's better to fill 680 * them with zeros here. 681 */ 682 if (count == 1) 683 cgi[0] = 0; 684 else 685 memset(cgi, 0, sizeof cgi); 686 687 while (count) { 688 batch = min_t(unsigned long, count, KPAGEFLAGS_BATCH); 689 pages = kpageflags_read(buf, index, batch); 690 if (pages == 0) 691 break; 692 693 if (kpagecgroup_read(cgi, index, pages) != pages) 694 fatal("kpagecgroup returned fewer pages than expected"); 695 696 if (kpagecount_read(cnt, index, pages) != pages) 697 fatal("kpagecount returned fewer pages than expected"); 698 699 for (i = 0; i < pages; i++) 700 add_page(voffset + i, index + i, 701 buf[i], cgi[i], cnt[i], pme); 702 703 index += pages; 704 count -= pages; 705 } 706 } 707 708 static void walk_swap(unsigned long voffset, uint64_t pme) 709 { 710 uint64_t flags = kpageflags_flags(0, pme); 711 712 if (!bit_mask_ok(flags)) 713 return; 714 715 if (opt_cgroup) 716 return; 717 718 if (opt_list == 1) 719 show_page_range(voffset, pagemap_swap_offset(pme), 720 1, flags, 0, 0); 721 else if (opt_list == 2) 722 show_page(voffset, pagemap_swap_offset(pme), flags, 0, 0); 723 724 nr_pages[hash_slot(flags)]++; 725 total_pages++; 726 } 727 728 #define PAGEMAP_BATCH (64 << 10) 729 static void walk_vma(unsigned long index, unsigned long count) 730 { 731 uint64_t buf[PAGEMAP_BATCH]; 732 unsigned long batch; 733 unsigned long pages; 734 unsigned long pfn; 735 unsigned long i; 736 737 while (count) { 738 batch = min_t(unsigned long, count, PAGEMAP_BATCH); 739 pages = pagemap_read(buf, index, batch); 740 if (pages == 0) 741 break; 742 743 for (i = 0; i < pages; i++) { 744 pfn = pagemap_pfn(buf[i]); 745 if (pfn) 746 walk_pfn(index + i, pfn, 1, buf[i]); 747 if (buf[i] & PM_SWAP) 748 walk_swap(index + i, buf[i]); 749 } 750 751 index += pages; 752 count -= pages; 753 } 754 } 755 756 static void walk_task(unsigned long index, unsigned long count) 757 { 758 const unsigned long end = index + count; 759 unsigned long start; 760 int i = 0; 761 762 while (index < end) { 763 764 while (pg_end[i] <= index) 765 if (++i >= nr_vmas) 766 return; 767 if (pg_start[i] >= end) 768 return; 769 770 start = max_t(unsigned long, pg_start[i], index); 771 index = min_t(unsigned long, pg_end[i], end); 772 773 assert(start < index); 774 walk_vma(start, index - start); 775 } 776 } 777 778 static void add_addr_range(unsigned long offset, unsigned long size) 779 { 780 if (nr_addr_ranges >= MAX_ADDR_RANGES) 781 fatal("too many addr ranges\n"); 782 783 opt_offset[nr_addr_ranges] = offset; 784 opt_size[nr_addr_ranges] = min_t(unsigned long, size, ULONG_MAX-offset); 785 nr_addr_ranges++; 786 } 787 788 static void walk_addr_ranges(void) 789 { 790 int i; 791 792 kpageflags_fd = checked_open(opt_kpageflags, O_RDONLY); 793 794 if (!nr_addr_ranges) 795 add_addr_range(0, ULONG_MAX); 796 797 for (i = 0; i < nr_addr_ranges; i++) 798 if (!opt_pid) 799 walk_pfn(opt_offset[i], opt_offset[i], opt_size[i], 0); 800 else 801 walk_task(opt_offset[i], opt_size[i]); 802 803 if (opt_mark_idle) 804 mark_page_idle(0); 805 806 close(kpageflags_fd); 807 } 808 809 810 /* 811 * user interface 812 */ 813 814 static const char *page_flag_type(uint64_t flag) 815 { 816 if (flag & KPF_HACKERS_BITS) 817 return "(r)"; 818 if (flag & KPF_OVERLOADED_BITS) 819 return "(o)"; 820 return " "; 821 } 822 823 static void usage(void) 824 { 825 size_t i, j; 826 827 printf( 828 "page-types [options]\n" 829 " -r|--raw Raw mode, for kernel developers\n" 830 " -d|--describe flags Describe flags\n" 831 " -a|--addr addr-spec Walk a range of pages\n" 832 " -b|--bits bits-spec Walk pages with specified bits\n" 833 " -c|--cgroup path|@inode Walk pages within memory cgroup\n" 834 " -p|--pid pid Walk process address space\n" 835 " -f|--file filename Walk file address space\n" 836 " -i|--mark-idle Mark pages idle\n" 837 " -l|--list Show page details in ranges\n" 838 " -L|--list-each Show page details one by one\n" 839 " -C|--list-cgroup Show cgroup inode for pages\n" 840 " -M|--list-mapcnt Show page map count\n" 841 " -N|--no-summary Don't show summary info\n" 842 " -X|--hwpoison hwpoison pages\n" 843 " -x|--unpoison unpoison pages\n" 844 " -F|--kpageflags filename kpageflags file to parse\n" 845 " -h|--help Show this usage message\n" 846 "flags:\n" 847 " 0x10 bitfield format, e.g.\n" 848 " anon bit-name, e.g.\n" 849 " 0x10,anon comma-separated list, e.g.\n" 850 "addr-spec:\n" 851 " N one page at offset N (unit: pages)\n" 852 " N+M pages range from N to N+M-1\n" 853 " N,M pages range from N to M-1\n" 854 " N, pages range from N to end\n" 855 " ,M pages range from 0 to M-1\n" 856 "bits-spec:\n" 857 " bit1,bit2 (flags & (bit1|bit2)) != 0\n" 858 " bit1,bit2=bit1 (flags & (bit1|bit2)) == bit1\n" 859 " bit1,~bit2 (flags & (bit1|bit2)) == bit1\n" 860 " =bit1,bit2 flags == (bit1|bit2)\n" 861 "bit-names:\n" 862 ); 863 864 for (i = 0, j = 0; i < ARRAY_SIZE(page_flag_names); i++) { 865 if (!page_flag_names[i]) 866 continue; 867 printf("%16s%s", page_flag_names[i] + 2, 868 page_flag_type(1ULL << i)); 869 if (++j > 3) { 870 j = 0; 871 putchar('\n'); 872 } 873 } 874 printf("\n " 875 "(r) raw mode bits (o) overloaded bits\n"); 876 } 877 878 static unsigned long long parse_number(const char *str) 879 { 880 unsigned long long n; 881 882 n = strtoll(str, NULL, 0); 883 884 if (n == 0 && str[0] != '0') 885 fatal("invalid name or number: %s\n", str); 886 887 return n; 888 } 889 890 static void parse_pid(const char *str) 891 { 892 FILE *file; 893 char buf[5000]; 894 895 opt_pid = parse_number(str); 896 897 sprintf(buf, "/proc/%d/pagemap", opt_pid); 898 pagemap_fd = checked_open(buf, O_RDONLY); 899 900 sprintf(buf, "/proc/%d/maps", opt_pid); 901 file = fopen(buf, "r"); 902 if (!file) { 903 perror(buf); 904 exit(EXIT_FAILURE); 905 } 906 907 while (fgets(buf, sizeof(buf), file) != NULL) { 908 unsigned long vm_start; 909 unsigned long vm_end; 910 unsigned long long pgoff; 911 int major, minor; 912 char r, w, x, s; 913 unsigned long ino; 914 int n; 915 916 n = sscanf(buf, "%lx-%lx %c%c%c%c %llx %x:%x %lu", 917 &vm_start, 918 &vm_end, 919 &r, &w, &x, &s, 920 &pgoff, 921 &major, &minor, 922 &ino); 923 if (n < 10) { 924 fprintf(stderr, "unexpected line: %s\n", buf); 925 continue; 926 } 927 pg_start[nr_vmas] = vm_start / page_size; 928 pg_end[nr_vmas] = vm_end / page_size; 929 if (++nr_vmas >= MAX_VMAS) { 930 fprintf(stderr, "too many VMAs\n"); 931 break; 932 } 933 } 934 fclose(file); 935 } 936 937 static void show_file(const char *name, const struct stat *st) 938 { 939 unsigned long long size = st->st_size; 940 char atime[64], mtime[64]; 941 long now = time(NULL); 942 943 printf("%s\tInode: %u\tSize: %llu (%llu pages)\n", 944 name, (unsigned)st->st_ino, 945 size, (size + page_size - 1) / page_size); 946 947 strftime(atime, sizeof(atime), "%c", localtime(&st->st_atime)); 948 strftime(mtime, sizeof(mtime), "%c", localtime(&st->st_mtime)); 949 950 printf("Modify: %s (%ld seconds ago)\nAccess: %s (%ld seconds ago)\n", 951 mtime, now - st->st_mtime, 952 atime, now - st->st_atime); 953 } 954 955 static sigjmp_buf sigbus_jmp; 956 957 static void * volatile sigbus_addr; 958 959 static void sigbus_handler(int sig, siginfo_t *info, void *ucontex) 960 { 961 (void)sig; 962 (void)ucontex; 963 sigbus_addr = info ? info->si_addr : NULL; 964 siglongjmp(sigbus_jmp, 1); 965 } 966 967 static struct sigaction sigbus_action = { 968 .sa_sigaction = sigbus_handler, 969 .sa_flags = SA_SIGINFO, 970 }; 971 972 static void walk_file_range(const char *name, int fd, 973 unsigned long off, unsigned long end) 974 { 975 uint8_t vec[PAGEMAP_BATCH]; 976 uint64_t buf[PAGEMAP_BATCH], flags; 977 uint64_t cgroup = 0; 978 uint64_t mapcnt = 0; 979 unsigned long nr_pages, pfn, i; 980 ssize_t len; 981 void *ptr; 982 int first = 1; 983 984 for (; off < end; off += len) { 985 nr_pages = (end - off + page_size - 1) / page_size; 986 if (nr_pages > PAGEMAP_BATCH) 987 nr_pages = PAGEMAP_BATCH; 988 len = nr_pages * page_size; 989 990 ptr = mmap(NULL, len, PROT_READ, MAP_SHARED, fd, off); 991 if (ptr == MAP_FAILED) 992 fatal("mmap failed: %s", name); 993 994 /* determine cached pages */ 995 if (mincore(ptr, len, vec)) 996 fatal("mincore failed: %s", name); 997 998 /* turn off readahead */ 999 if (madvise(ptr, len, MADV_RANDOM)) 1000 fatal("madvice failed: %s", name); 1001 1002 if (sigsetjmp(sigbus_jmp, 1)) { 1003 end = off + sigbus_addr ? sigbus_addr - ptr : 0; 1004 fprintf(stderr, "got sigbus at offset %lld: %s\n", 1005 (long long)end, name); 1006 goto got_sigbus; 1007 } 1008 1009 /* populate ptes */ 1010 for (i = 0; i < nr_pages ; i++) { 1011 if (vec[i] & 1) 1012 (void)*(volatile int *)(ptr + i * page_size); 1013 } 1014 got_sigbus: 1015 1016 /* turn off harvesting reference bits */ 1017 if (madvise(ptr, len, MADV_SEQUENTIAL)) 1018 fatal("madvice failed: %s", name); 1019 1020 if (pagemap_read(buf, (unsigned long)ptr / page_size, 1021 nr_pages) != nr_pages) 1022 fatal("cannot read pagemap"); 1023 1024 munmap(ptr, len); 1025 1026 for (i = 0; i < nr_pages; i++) { 1027 pfn = pagemap_pfn(buf[i]); 1028 if (!pfn) 1029 continue; 1030 if (!kpageflags_read(&flags, pfn, 1)) 1031 continue; 1032 if (!kpagecgroup_read(&cgroup, pfn, 1)) 1033 fatal("kpagecgroup_read failed"); 1034 if (!kpagecount_read(&mapcnt, pfn, 1)) 1035 fatal("kpagecount_read failed"); 1036 if (first && opt_list) { 1037 first = 0; 1038 flush_page_range(); 1039 } 1040 add_page(off / page_size + i, pfn, 1041 flags, cgroup, mapcnt, buf[i]); 1042 } 1043 } 1044 } 1045 1046 static void walk_file(const char *name, const struct stat *st) 1047 { 1048 int i; 1049 int fd; 1050 1051 fd = checked_open(name, O_RDONLY|O_NOATIME|O_NOFOLLOW); 1052 1053 if (!nr_addr_ranges) 1054 add_addr_range(0, st->st_size / page_size); 1055 1056 for (i = 0; i < nr_addr_ranges; i++) 1057 walk_file_range(name, fd, opt_offset[i] * page_size, 1058 (opt_offset[i] + opt_size[i]) * page_size); 1059 1060 close(fd); 1061 } 1062 1063 int walk_tree(const char *name, const struct stat *st, int type, struct FTW *f) 1064 { 1065 (void)f; 1066 switch (type) { 1067 case FTW_F: 1068 if (S_ISREG(st->st_mode)) 1069 walk_file(name, st); 1070 break; 1071 case FTW_DNR: 1072 fprintf(stderr, "cannot read dir: %s\n", name); 1073 break; 1074 } 1075 return 0; 1076 } 1077 1078 struct stat st; 1079 1080 static void walk_page_cache(void) 1081 { 1082 kpageflags_fd = checked_open(opt_kpageflags, O_RDONLY); 1083 pagemap_fd = checked_open("/proc/self/pagemap", O_RDONLY); 1084 sigaction(SIGBUS, &sigbus_action, NULL); 1085 1086 if (stat(opt_file, &st)) 1087 fatal("stat failed: %s\n", opt_file); 1088 1089 if (S_ISREG(st.st_mode)) { 1090 walk_file(opt_file, &st); 1091 } else if (S_ISDIR(st.st_mode)) { 1092 /* do not follow symlinks and mountpoints */ 1093 if (nftw(opt_file, walk_tree, 64, FTW_MOUNT | FTW_PHYS) < 0) 1094 fatal("nftw failed: %s\n", opt_file); 1095 } else 1096 fatal("unhandled file type: %s\n", opt_file); 1097 1098 close(kpageflags_fd); 1099 close(pagemap_fd); 1100 signal(SIGBUS, SIG_DFL); 1101 } 1102 1103 static void parse_file(const char *name) 1104 { 1105 opt_file = name; 1106 } 1107 1108 static void parse_cgroup(const char *path) 1109 { 1110 if (path[0] == '@') { 1111 opt_cgroup = parse_number(path + 1); 1112 return; 1113 } 1114 1115 struct stat st; 1116 1117 if (stat(path, &st)) 1118 fatal("stat failed: %s: %m\n", path); 1119 1120 if (!S_ISDIR(st.st_mode)) 1121 fatal("cgroup supposed to be a directory: %s\n", path); 1122 1123 opt_cgroup = st.st_ino; 1124 } 1125 1126 static void parse_addr_range(const char *optarg) 1127 { 1128 unsigned long offset; 1129 unsigned long size; 1130 char *p; 1131 1132 p = strchr(optarg, ','); 1133 if (!p) 1134 p = strchr(optarg, '+'); 1135 1136 if (p == optarg) { 1137 offset = 0; 1138 size = parse_number(p + 1); 1139 } else if (p) { 1140 offset = parse_number(optarg); 1141 if (p[1] == '\0') 1142 size = ULONG_MAX; 1143 else { 1144 size = parse_number(p + 1); 1145 if (*p == ',') { 1146 if (size < offset) 1147 fatal("invalid range: %lu,%lu\n", 1148 offset, size); 1149 size -= offset; 1150 } 1151 } 1152 } else { 1153 offset = parse_number(optarg); 1154 size = 1; 1155 } 1156 1157 add_addr_range(offset, size); 1158 } 1159 1160 static void add_bits_filter(uint64_t mask, uint64_t bits) 1161 { 1162 if (nr_bit_filters >= MAX_BIT_FILTERS) 1163 fatal("too much bit filters\n"); 1164 1165 opt_mask[nr_bit_filters] = mask; 1166 opt_bits[nr_bit_filters] = bits; 1167 nr_bit_filters++; 1168 } 1169 1170 static uint64_t parse_flag_name(const char *str, int len) 1171 { 1172 size_t i; 1173 1174 if (!*str || !len) 1175 return 0; 1176 1177 if (len <= 8 && !strncmp(str, "compound", len)) 1178 return BITS_COMPOUND; 1179 1180 for (i = 0; i < ARRAY_SIZE(page_flag_names); i++) { 1181 if (!page_flag_names[i]) 1182 continue; 1183 if (!strncmp(str, page_flag_names[i] + 2, len)) 1184 return 1ULL << i; 1185 } 1186 1187 return parse_number(str); 1188 } 1189 1190 static uint64_t parse_flag_names(const char *str, int all) 1191 { 1192 const char *p = str; 1193 uint64_t flags = 0; 1194 1195 while (1) { 1196 if (*p == ',' || *p == '=' || *p == '\0') { 1197 if ((*str != '~') || (*str == '~' && all && *++str)) 1198 flags |= parse_flag_name(str, p - str); 1199 if (*p != ',') 1200 break; 1201 str = p + 1; 1202 } 1203 p++; 1204 } 1205 1206 return flags; 1207 } 1208 1209 static void parse_bits_mask(const char *optarg) 1210 { 1211 uint64_t mask; 1212 uint64_t bits; 1213 const char *p; 1214 1215 p = strchr(optarg, '='); 1216 if (p == optarg) { 1217 mask = KPF_ALL_BITS; 1218 bits = parse_flag_names(p + 1, 0); 1219 } else if (p) { 1220 mask = parse_flag_names(optarg, 0); 1221 bits = parse_flag_names(p + 1, 0); 1222 } else if (strchr(optarg, '~')) { 1223 mask = parse_flag_names(optarg, 1); 1224 bits = parse_flag_names(optarg, 0); 1225 } else { 1226 mask = parse_flag_names(optarg, 0); 1227 bits = KPF_ALL_BITS; 1228 } 1229 1230 add_bits_filter(mask, bits); 1231 } 1232 1233 static void parse_kpageflags(const char *name) 1234 { 1235 opt_kpageflags = name; 1236 } 1237 1238 static void describe_flags(const char *optarg) 1239 { 1240 uint64_t flags = parse_flag_names(optarg, 0); 1241 1242 printf("0x%016llx\t%s\t%s\n", 1243 (unsigned long long)flags, 1244 page_flag_name(flags), 1245 page_flag_longname(flags)); 1246 } 1247 1248 static const struct option opts[] = { 1249 { "raw" , 0, NULL, 'r' }, 1250 { "pid" , 1, NULL, 'p' }, 1251 { "file" , 1, NULL, 'f' }, 1252 { "addr" , 1, NULL, 'a' }, 1253 { "bits" , 1, NULL, 'b' }, 1254 { "cgroup" , 1, NULL, 'c' }, 1255 { "describe" , 1, NULL, 'd' }, 1256 { "mark-idle" , 0, NULL, 'i' }, 1257 { "list" , 0, NULL, 'l' }, 1258 { "list-each" , 0, NULL, 'L' }, 1259 { "list-cgroup", 0, NULL, 'C' }, 1260 { "list-mapcnt", 0, NULL, 'M' }, 1261 { "no-summary", 0, NULL, 'N' }, 1262 { "hwpoison" , 0, NULL, 'X' }, 1263 { "unpoison" , 0, NULL, 'x' }, 1264 { "kpageflags", 0, NULL, 'F' }, 1265 { "help" , 0, NULL, 'h' }, 1266 { NULL , 0, NULL, 0 } 1267 }; 1268 1269 int main(int argc, char *argv[]) 1270 { 1271 int c; 1272 1273 page_size = getpagesize(); 1274 1275 while ((c = getopt_long(argc, argv, 1276 "rp:f:a:b:d:c:CilLMNXxF:h", 1277 opts, NULL)) != -1) { 1278 switch (c) { 1279 case 'r': 1280 opt_raw = 1; 1281 break; 1282 case 'p': 1283 parse_pid(optarg); 1284 break; 1285 case 'f': 1286 parse_file(optarg); 1287 break; 1288 case 'a': 1289 parse_addr_range(optarg); 1290 break; 1291 case 'b': 1292 parse_bits_mask(optarg); 1293 break; 1294 case 'c': 1295 parse_cgroup(optarg); 1296 break; 1297 case 'C': 1298 opt_list_cgroup = 1; 1299 break; 1300 case 'd': 1301 describe_flags(optarg); 1302 exit(0); 1303 case 'i': 1304 opt_mark_idle = 1; 1305 break; 1306 case 'l': 1307 opt_list = 1; 1308 break; 1309 case 'L': 1310 opt_list = 2; 1311 break; 1312 case 'M': 1313 opt_list_mapcnt = 1; 1314 break; 1315 case 'N': 1316 opt_no_summary = 1; 1317 break; 1318 case 'X': 1319 opt_hwpoison = 1; 1320 prepare_hwpoison_fd(); 1321 break; 1322 case 'x': 1323 opt_unpoison = 1; 1324 prepare_hwpoison_fd(); 1325 break; 1326 case 'F': 1327 parse_kpageflags(optarg); 1328 break; 1329 case 'h': 1330 usage(); 1331 exit(0); 1332 default: 1333 usage(); 1334 exit(1); 1335 } 1336 } 1337 1338 if (!opt_kpageflags) 1339 opt_kpageflags = PROC_KPAGEFLAGS; 1340 1341 if (opt_cgroup || opt_list_cgroup) 1342 kpagecgroup_fd = checked_open(PROC_KPAGECGROUP, O_RDONLY); 1343 1344 if (opt_list && opt_list_mapcnt) 1345 kpagecount_fd = checked_open(PROC_KPAGECOUNT, O_RDONLY); 1346 1347 if (opt_mark_idle) 1348 page_idle_fd = checked_open(SYS_KERNEL_MM_PAGE_IDLE, O_RDWR); 1349 1350 if (opt_list && opt_pid) 1351 printf("voffset\t"); 1352 if (opt_list && opt_file) 1353 printf("foffset\t"); 1354 if (opt_list && opt_list_cgroup) 1355 printf("cgroup\t"); 1356 if (opt_list && opt_list_mapcnt) 1357 printf("map-cnt\t"); 1358 1359 if (opt_list == 1) 1360 printf("offset\tlen\tflags\n"); 1361 if (opt_list == 2) 1362 printf("offset\tflags\n"); 1363 1364 if (opt_file) 1365 walk_page_cache(); 1366 else 1367 walk_addr_ranges(); 1368 1369 if (opt_list == 1) 1370 flush_page_range(); 1371 1372 if (opt_no_summary) 1373 return 0; 1374 1375 if (opt_list) 1376 printf("\n\n"); 1377 1378 if (opt_file) { 1379 show_file(opt_file, &st); 1380 printf("\n"); 1381 } 1382 1383 show_summary(); 1384 1385 if (opt_list_mapcnt) 1386 close(kpagecount_fd); 1387 1388 if (page_idle_fd >= 0) 1389 close(page_idle_fd); 1390 1391 return 0; 1392 } 1393