1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * page-types: Tool for querying page flags 4 * 5 * Copyright (C) 2009 Intel corporation 6 * 7 * Authors: Wu Fengguang <fengguang.wu@intel.com> 8 */ 9 10 #define _FILE_OFFSET_BITS 64 11 #define _GNU_SOURCE 12 #include <stdio.h> 13 #include <stdlib.h> 14 #include <unistd.h> 15 #include <stdint.h> 16 #include <stdarg.h> 17 #include <string.h> 18 #include <getopt.h> 19 #include <limits.h> 20 #include <assert.h> 21 #include <ftw.h> 22 #include <time.h> 23 #include <setjmp.h> 24 #include <signal.h> 25 #include <inttypes.h> 26 #include <sys/types.h> 27 #include <errno.h> 28 #include <fcntl.h> 29 #include <sys/mount.h> 30 #include <sys/statfs.h> 31 #include <sys/mman.h> 32 #include "../../include/uapi/linux/magic.h" 33 #include "../../include/uapi/linux/kernel-page-flags.h" 34 #include <api/fs/fs.h> 35 36 #ifndef MAX_PATH 37 # define MAX_PATH 256 38 #endif 39 40 #ifndef STR 41 # define _STR(x) #x 42 # define STR(x) _STR(x) 43 #endif 44 45 /* 46 * pagemap kernel ABI bits 47 */ 48 49 #define PM_ENTRY_BYTES 8 50 #define PM_PFRAME_BITS 55 51 #define PM_PFRAME_MASK ((1LL << PM_PFRAME_BITS) - 1) 52 #define PM_PFRAME(x) ((x) & PM_PFRAME_MASK) 53 #define MAX_SWAPFILES_SHIFT 5 54 #define PM_SWAP_OFFSET(x) (((x) & PM_PFRAME_MASK) >> MAX_SWAPFILES_SHIFT) 55 #define PM_SOFT_DIRTY (1ULL << 55) 56 #define PM_MMAP_EXCLUSIVE (1ULL << 56) 57 #define PM_FILE (1ULL << 61) 58 #define PM_SWAP (1ULL << 62) 59 #define PM_PRESENT (1ULL << 63) 60 61 /* 62 * kernel page flags 63 */ 64 65 #define KPF_BYTES 8 66 #define PROC_KPAGEFLAGS "/proc/kpageflags" 67 #define PROC_KPAGECOUNT "/proc/kpagecount" 68 #define PROC_KPAGECGROUP "/proc/kpagecgroup" 69 70 #define SYS_KERNEL_MM_PAGE_IDLE "/sys/kernel/mm/page_idle/bitmap" 71 72 /* [32-] kernel hacking assistances */ 73 #define KPF_RESERVED 32 74 #define KPF_MLOCKED 33 75 #define KPF_OWNER_2 34 76 #define KPF_PRIVATE_2 36 77 #define KPF_OWNER_PRIVATE 37 78 #define KPF_ARCH 38 79 #define KPF_UNCACHED 39 /* unused */ 80 #define KPF_SOFTDIRTY 40 81 #define KPF_ARCH_2 41 82 83 /* [47-] take some arbitrary free slots for expanding overloaded flags 84 * not part of kernel API 85 */ 86 #define KPF_ANON_EXCLUSIVE 47 87 #define KPF_READAHEAD 48 88 #define KPF_SLUB_FROZEN 50 89 #define KPF_SLUB_DEBUG 51 90 #define KPF_FILE 61 91 #define KPF_SWAP 62 92 #define KPF_MMAP_EXCLUSIVE 63 93 94 #define KPF_ALL_BITS ((uint64_t)~0ULL) 95 #define KPF_HACKERS_BITS (0xffffULL << 32) 96 #define KPF_OVERLOADED_BITS (0xffffULL << 48) 97 #define BIT(name) (1ULL << KPF_##name) 98 #define BITS_COMPOUND (BIT(COMPOUND_HEAD) | BIT(COMPOUND_TAIL)) 99 100 static const char * const page_flag_names[] = { 101 [KPF_LOCKED] = "L:locked", 102 [KPF_ERROR] = "E:error", 103 [KPF_REFERENCED] = "R:referenced", 104 [KPF_UPTODATE] = "U:uptodate", 105 [KPF_DIRTY] = "D:dirty", 106 [KPF_LRU] = "l:lru", 107 [KPF_ACTIVE] = "A:active", 108 [KPF_SLAB] = "S:slab", 109 [KPF_WRITEBACK] = "W:writeback", 110 [KPF_RECLAIM] = "I:reclaim", 111 [KPF_BUDDY] = "B:buddy", 112 113 [KPF_MMAP] = "M:mmap", 114 [KPF_ANON] = "a:anonymous", 115 [KPF_SWAPCACHE] = "s:swapcache", 116 [KPF_SWAPBACKED] = "b:swapbacked", 117 [KPF_COMPOUND_HEAD] = "H:compound_head", 118 [KPF_COMPOUND_TAIL] = "T:compound_tail", 119 [KPF_HUGE] = "G:huge", 120 [KPF_UNEVICTABLE] = "u:unevictable", 121 [KPF_HWPOISON] = "X:hwpoison", 122 [KPF_NOPAGE] = "n:nopage", 123 [KPF_KSM] = "x:ksm", 124 [KPF_THP] = "t:thp", 125 [KPF_OFFLINE] = "o:offline", 126 [KPF_PGTABLE] = "g:pgtable", 127 [KPF_ZERO_PAGE] = "z:zero_page", 128 [KPF_IDLE] = "i:idle_page", 129 130 [KPF_RESERVED] = "r:reserved", 131 [KPF_MLOCKED] = "m:mlocked", 132 [KPF_OWNER_2] = "d:owner_2", 133 [KPF_PRIVATE_2] = "p:private_2", 134 [KPF_OWNER_PRIVATE] = "O:owner_private", 135 [KPF_ARCH] = "h:arch", 136 [KPF_SOFTDIRTY] = "f:softdirty", 137 [KPF_ARCH_2] = "H:arch_2", 138 139 [KPF_ANON_EXCLUSIVE] = "d:anon_exclusive", 140 [KPF_READAHEAD] = "I:readahead", 141 [KPF_SLUB_FROZEN] = "A:slub_frozen", 142 [KPF_SLUB_DEBUG] = "E:slub_debug", 143 144 [KPF_FILE] = "F:file", 145 [KPF_SWAP] = "w:swap", 146 [KPF_MMAP_EXCLUSIVE] = "1:mmap_exclusive", 147 }; 148 149 150 /* 151 * data structures 152 */ 153 154 static int opt_raw; /* for kernel developers */ 155 static int opt_list; /* list pages (in ranges) */ 156 static int opt_mark_idle; /* set accessed bit */ 157 static int opt_no_summary; /* don't show summary */ 158 static pid_t opt_pid; /* process to walk */ 159 const char *opt_file; /* file or directory path */ 160 static uint64_t opt_cgroup; /* cgroup inode */ 161 static int opt_list_cgroup;/* list page cgroup */ 162 static int opt_list_mapcnt;/* list page map count */ 163 static const char *opt_kpageflags;/* kpageflags file to parse */ 164 165 #define MAX_ADDR_RANGES 1024 166 static int nr_addr_ranges; 167 static unsigned long opt_offset[MAX_ADDR_RANGES]; 168 static unsigned long opt_size[MAX_ADDR_RANGES]; 169 170 #define MAX_VMAS 10240 171 static int nr_vmas; 172 static unsigned long pg_start[MAX_VMAS]; 173 static unsigned long pg_end[MAX_VMAS]; 174 175 #define MAX_BIT_FILTERS 64 176 static int nr_bit_filters; 177 static uint64_t opt_mask[MAX_BIT_FILTERS]; 178 static uint64_t opt_bits[MAX_BIT_FILTERS]; 179 180 static int page_size; 181 182 static int pagemap_fd; 183 static int kpageflags_fd; 184 static int kpagecount_fd = -1; 185 static int kpagecgroup_fd = -1; 186 static int page_idle_fd = -1; 187 188 static int opt_hwpoison; 189 static int opt_unpoison; 190 191 static const char *hwpoison_debug_fs; 192 static int hwpoison_inject_fd; 193 static int hwpoison_forget_fd; 194 195 #define HASH_SHIFT 13 196 #define HASH_SIZE (1 << HASH_SHIFT) 197 #define HASH_MASK (HASH_SIZE - 1) 198 #define HASH_KEY(flags) (flags & HASH_MASK) 199 200 static unsigned long total_pages; 201 static unsigned long nr_pages[HASH_SIZE]; 202 static uint64_t page_flags[HASH_SIZE]; 203 204 205 /* 206 * helper functions 207 */ 208 209 #define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) 210 211 #define min_t(type, x, y) ({ \ 212 type __min1 = (x); \ 213 type __min2 = (y); \ 214 __min1 < __min2 ? __min1 : __min2; }) 215 216 #define max_t(type, x, y) ({ \ 217 type __max1 = (x); \ 218 type __max2 = (y); \ 219 __max1 > __max2 ? __max1 : __max2; }) 220 221 static unsigned long pages2mb(unsigned long pages) 222 { 223 return (pages * page_size) >> 20; 224 } 225 226 static void fatal(const char *x, ...) 227 { 228 va_list ap; 229 230 va_start(ap, x); 231 vfprintf(stderr, x, ap); 232 va_end(ap); 233 exit(EXIT_FAILURE); 234 } 235 236 static int checked_open(const char *pathname, int flags) 237 { 238 int fd = open(pathname, flags); 239 240 if (fd < 0) { 241 perror(pathname); 242 exit(EXIT_FAILURE); 243 } 244 245 return fd; 246 } 247 248 /* 249 * pagemap/kpageflags routines 250 */ 251 252 static unsigned long do_u64_read(int fd, const char *name, 253 uint64_t *buf, 254 unsigned long index, 255 unsigned long count) 256 { 257 long bytes; 258 259 if (index > ULONG_MAX / 8) 260 fatal("index overflow: %lu\n", index); 261 262 bytes = pread(fd, buf, count * 8, (off_t)index * 8); 263 if (bytes < 0) { 264 perror(name); 265 exit(EXIT_FAILURE); 266 } 267 if (bytes % 8) 268 fatal("partial read: %lu bytes\n", bytes); 269 270 return bytes / 8; 271 } 272 273 static unsigned long kpageflags_read(uint64_t *buf, 274 unsigned long index, 275 unsigned long pages) 276 { 277 return do_u64_read(kpageflags_fd, opt_kpageflags, buf, index, pages); 278 } 279 280 static unsigned long kpagecgroup_read(uint64_t *buf, 281 unsigned long index, 282 unsigned long pages) 283 { 284 if (kpagecgroup_fd < 0) 285 return pages; 286 287 return do_u64_read(kpagecgroup_fd, opt_kpageflags, buf, index, pages); 288 } 289 290 static unsigned long kpagecount_read(uint64_t *buf, 291 unsigned long index, 292 unsigned long pages) 293 { 294 return kpagecount_fd < 0 ? pages : 295 do_u64_read(kpagecount_fd, PROC_KPAGECOUNT, 296 buf, index, pages); 297 } 298 299 static unsigned long pagemap_read(uint64_t *buf, 300 unsigned long index, 301 unsigned long pages) 302 { 303 return do_u64_read(pagemap_fd, "/proc/pid/pagemap", buf, index, pages); 304 } 305 306 static unsigned long pagemap_pfn(uint64_t val) 307 { 308 unsigned long pfn; 309 310 if (val & PM_PRESENT) 311 pfn = PM_PFRAME(val); 312 else 313 pfn = 0; 314 315 return pfn; 316 } 317 318 static unsigned long pagemap_swap_offset(uint64_t val) 319 { 320 return val & PM_SWAP ? PM_SWAP_OFFSET(val) : 0; 321 } 322 323 /* 324 * page flag names 325 */ 326 327 static char *page_flag_name(uint64_t flags) 328 { 329 static char buf[65]; 330 int present; 331 size_t i, j; 332 333 for (i = 0, j = 0; i < ARRAY_SIZE(page_flag_names); i++) { 334 present = (flags >> i) & 1; 335 if (!page_flag_names[i]) { 336 if (present) 337 fatal("unknown flag bit %d\n", i); 338 continue; 339 } 340 buf[j++] = present ? page_flag_names[i][0] : '_'; 341 } 342 343 return buf; 344 } 345 346 static char *page_flag_longname(uint64_t flags) 347 { 348 static char buf[1024]; 349 size_t i, n; 350 351 for (i = 0, n = 0; i < ARRAY_SIZE(page_flag_names); i++) { 352 if (!page_flag_names[i]) 353 continue; 354 if ((flags >> i) & 1) 355 n += snprintf(buf + n, sizeof(buf) - n, "%s,", 356 page_flag_names[i] + 2); 357 } 358 if (n) 359 n--; 360 buf[n] = '\0'; 361 362 return buf; 363 } 364 365 366 /* 367 * page list and summary 368 */ 369 370 static void show_page_range(unsigned long voffset, unsigned long offset, 371 unsigned long size, uint64_t flags, 372 uint64_t cgroup, uint64_t mapcnt) 373 { 374 static uint64_t flags0; 375 static uint64_t cgroup0; 376 static uint64_t mapcnt0; 377 static unsigned long voff; 378 static unsigned long index; 379 static unsigned long count; 380 381 if (flags == flags0 && cgroup == cgroup0 && mapcnt == mapcnt0 && 382 offset == index + count && size && voffset == voff + count) { 383 count += size; 384 return; 385 } 386 387 if (count) { 388 if (opt_pid) 389 printf("%lx\t", voff); 390 if (opt_file) 391 printf("%lx\t", voff); 392 if (opt_list_cgroup) 393 printf("@%" PRIu64 "\t", cgroup0); 394 if (opt_list_mapcnt) 395 printf("%" PRIu64 "\t", mapcnt0); 396 printf("%lx\t%lx\t%s\n", 397 index, count, page_flag_name(flags0)); 398 } 399 400 flags0 = flags; 401 cgroup0 = cgroup; 402 mapcnt0 = mapcnt; 403 index = offset; 404 voff = voffset; 405 count = size; 406 } 407 408 static void flush_page_range(void) 409 { 410 show_page_range(0, 0, 0, 0, 0, 0); 411 } 412 413 static void show_page(unsigned long voffset, unsigned long offset, 414 uint64_t flags, uint64_t cgroup, uint64_t mapcnt) 415 { 416 if (opt_pid) 417 printf("%lx\t", voffset); 418 if (opt_file) 419 printf("%lx\t", voffset); 420 if (opt_list_cgroup) 421 printf("@%" PRIu64 "\t", cgroup); 422 if (opt_list_mapcnt) 423 printf("%" PRIu64 "\t", mapcnt); 424 425 printf("%lx\t%s\n", offset, page_flag_name(flags)); 426 } 427 428 static void show_summary(void) 429 { 430 size_t i; 431 432 printf(" flags\tpage-count MB" 433 " symbolic-flags\t\t\tlong-symbolic-flags\n"); 434 435 for (i = 0; i < ARRAY_SIZE(nr_pages); i++) { 436 if (nr_pages[i]) 437 printf("0x%016llx\t%10lu %8lu %s\t%s\n", 438 (unsigned long long)page_flags[i], 439 nr_pages[i], 440 pages2mb(nr_pages[i]), 441 page_flag_name(page_flags[i]), 442 page_flag_longname(page_flags[i])); 443 } 444 445 printf(" total\t%10lu %8lu\n", 446 total_pages, pages2mb(total_pages)); 447 } 448 449 450 /* 451 * page flag filters 452 */ 453 454 static int bit_mask_ok(uint64_t flags) 455 { 456 int i; 457 458 for (i = 0; i < nr_bit_filters; i++) { 459 if (opt_bits[i] == KPF_ALL_BITS) { 460 if ((flags & opt_mask[i]) == 0) 461 return 0; 462 } else { 463 if ((flags & opt_mask[i]) != opt_bits[i]) 464 return 0; 465 } 466 } 467 468 return 1; 469 } 470 471 static uint64_t expand_overloaded_flags(uint64_t flags, uint64_t pme) 472 { 473 /* Anonymous pages use PG_owner_2 for anon_exclusive */ 474 if ((flags & BIT(ANON)) && (flags & BIT(OWNER_2))) 475 flags ^= BIT(OWNER_2) | BIT(ANON_EXCLUSIVE); 476 477 /* SLUB overloads several page flags */ 478 if (flags & BIT(SLAB)) { 479 if (flags & BIT(ACTIVE)) 480 flags ^= BIT(ACTIVE) | BIT(SLUB_FROZEN); 481 if (flags & BIT(ERROR)) 482 flags ^= BIT(ERROR) | BIT(SLUB_DEBUG); 483 } 484 485 /* PG_reclaim is overloaded as PG_readahead in the read path */ 486 if ((flags & (BIT(RECLAIM) | BIT(WRITEBACK))) == BIT(RECLAIM)) 487 flags ^= BIT(RECLAIM) | BIT(READAHEAD); 488 489 if (pme & PM_SOFT_DIRTY) 490 flags |= BIT(SOFTDIRTY); 491 if (pme & PM_FILE) 492 flags |= BIT(FILE); 493 if (pme & PM_SWAP) 494 flags |= BIT(SWAP); 495 if (pme & PM_MMAP_EXCLUSIVE) 496 flags |= BIT(MMAP_EXCLUSIVE); 497 498 return flags; 499 } 500 501 static uint64_t well_known_flags(uint64_t flags) 502 { 503 /* hide flags intended only for kernel hacker */ 504 flags &= ~KPF_HACKERS_BITS; 505 506 /* hide non-hugeTLB compound pages */ 507 if ((flags & BITS_COMPOUND) && !(flags & BIT(HUGE))) 508 flags &= ~BITS_COMPOUND; 509 510 return flags; 511 } 512 513 static uint64_t kpageflags_flags(uint64_t flags, uint64_t pme) 514 { 515 if (opt_raw) 516 flags = expand_overloaded_flags(flags, pme); 517 else 518 flags = well_known_flags(flags); 519 520 return flags; 521 } 522 523 /* 524 * page actions 525 */ 526 527 static void prepare_hwpoison_fd(void) 528 { 529 char buf[MAX_PATH + 1]; 530 531 hwpoison_debug_fs = debugfs__mount(); 532 if (!hwpoison_debug_fs) { 533 perror("mount debugfs"); 534 exit(EXIT_FAILURE); 535 } 536 537 if (opt_hwpoison && !hwpoison_inject_fd) { 538 snprintf(buf, MAX_PATH, "%s/hwpoison/corrupt-pfn", 539 hwpoison_debug_fs); 540 hwpoison_inject_fd = checked_open(buf, O_WRONLY); 541 } 542 543 if (opt_unpoison && !hwpoison_forget_fd) { 544 snprintf(buf, MAX_PATH, "%s/hwpoison/unpoison-pfn", 545 hwpoison_debug_fs); 546 hwpoison_forget_fd = checked_open(buf, O_WRONLY); 547 } 548 } 549 550 static int hwpoison_page(unsigned long offset) 551 { 552 char buf[100]; 553 int len; 554 555 len = sprintf(buf, "0x%lx\n", offset); 556 len = write(hwpoison_inject_fd, buf, len); 557 if (len < 0) { 558 perror("hwpoison inject"); 559 return len; 560 } 561 return 0; 562 } 563 564 static int unpoison_page(unsigned long offset) 565 { 566 char buf[100]; 567 int len; 568 569 len = sprintf(buf, "0x%lx\n", offset); 570 len = write(hwpoison_forget_fd, buf, len); 571 if (len < 0) { 572 perror("hwpoison forget"); 573 return len; 574 } 575 return 0; 576 } 577 578 static int mark_page_idle(unsigned long offset) 579 { 580 static unsigned long off; 581 static uint64_t buf; 582 int len; 583 584 if ((offset / 64 == off / 64) || buf == 0) { 585 buf |= 1UL << (offset % 64); 586 off = offset; 587 return 0; 588 } 589 590 len = pwrite(page_idle_fd, &buf, 8, 8 * (off / 64)); 591 if (len < 0) { 592 perror("mark page idle"); 593 return len; 594 } 595 596 buf = 1UL << (offset % 64); 597 off = offset; 598 599 return 0; 600 } 601 602 /* 603 * page frame walker 604 */ 605 606 static size_t hash_slot(uint64_t flags) 607 { 608 size_t k = HASH_KEY(flags); 609 size_t i; 610 611 /* Explicitly reserve slot 0 for flags 0: the following logic 612 * cannot distinguish an unoccupied slot from slot (flags==0). 613 */ 614 if (flags == 0) 615 return 0; 616 617 /* search through the remaining (HASH_SIZE-1) slots */ 618 for (i = 1; i < ARRAY_SIZE(page_flags); i++, k++) { 619 if (!k || k >= ARRAY_SIZE(page_flags)) 620 k = 1; 621 if (page_flags[k] == 0) { 622 page_flags[k] = flags; 623 return k; 624 } 625 if (page_flags[k] == flags) 626 return k; 627 } 628 629 fatal("hash table full: bump up HASH_SHIFT?\n"); 630 exit(EXIT_FAILURE); 631 } 632 633 static void add_page(unsigned long voffset, unsigned long offset, 634 uint64_t flags, uint64_t cgroup, uint64_t mapcnt, 635 uint64_t pme) 636 { 637 flags = kpageflags_flags(flags, pme); 638 639 if (!bit_mask_ok(flags)) 640 return; 641 642 if (opt_cgroup && cgroup != (uint64_t)opt_cgroup) 643 return; 644 645 if (opt_hwpoison) 646 hwpoison_page(offset); 647 if (opt_unpoison) 648 unpoison_page(offset); 649 650 if (opt_mark_idle) 651 mark_page_idle(offset); 652 653 if (opt_list == 1) 654 show_page_range(voffset, offset, 1, flags, cgroup, mapcnt); 655 else if (opt_list == 2) 656 show_page(voffset, offset, flags, cgroup, mapcnt); 657 658 nr_pages[hash_slot(flags)]++; 659 total_pages++; 660 } 661 662 #define KPAGEFLAGS_BATCH (64 << 10) /* 64k pages */ 663 static void walk_pfn(unsigned long voffset, 664 unsigned long index, 665 unsigned long count, 666 uint64_t pme) 667 { 668 uint64_t buf[KPAGEFLAGS_BATCH]; 669 uint64_t cgi[KPAGEFLAGS_BATCH]; 670 uint64_t cnt[KPAGEFLAGS_BATCH]; 671 unsigned long batch; 672 unsigned long pages; 673 unsigned long i; 674 675 /* 676 * kpagecgroup_read() reads only if kpagecgroup were opened, but 677 * /proc/kpagecgroup might even not exist, so it's better to fill 678 * them with zeros here. 679 */ 680 if (count == 1) 681 cgi[0] = 0; 682 else 683 memset(cgi, 0, sizeof cgi); 684 685 while (count) { 686 batch = min_t(unsigned long, count, KPAGEFLAGS_BATCH); 687 pages = kpageflags_read(buf, index, batch); 688 if (pages == 0) 689 break; 690 691 if (kpagecgroup_read(cgi, index, pages) != pages) 692 fatal("kpagecgroup returned fewer pages than expected"); 693 694 if (kpagecount_read(cnt, index, pages) != pages) 695 fatal("kpagecount returned fewer pages than expected"); 696 697 for (i = 0; i < pages; i++) 698 add_page(voffset + i, index + i, 699 buf[i], cgi[i], cnt[i], pme); 700 701 index += pages; 702 count -= pages; 703 } 704 } 705 706 static void walk_swap(unsigned long voffset, uint64_t pme) 707 { 708 uint64_t flags = kpageflags_flags(0, pme); 709 710 if (!bit_mask_ok(flags)) 711 return; 712 713 if (opt_cgroup) 714 return; 715 716 if (opt_list == 1) 717 show_page_range(voffset, pagemap_swap_offset(pme), 718 1, flags, 0, 0); 719 else if (opt_list == 2) 720 show_page(voffset, pagemap_swap_offset(pme), flags, 0, 0); 721 722 nr_pages[hash_slot(flags)]++; 723 total_pages++; 724 } 725 726 #define PAGEMAP_BATCH (64 << 10) 727 static void walk_vma(unsigned long index, unsigned long count) 728 { 729 uint64_t buf[PAGEMAP_BATCH]; 730 unsigned long batch; 731 unsigned long pages; 732 unsigned long pfn; 733 unsigned long i; 734 735 while (count) { 736 batch = min_t(unsigned long, count, PAGEMAP_BATCH); 737 pages = pagemap_read(buf, index, batch); 738 if (pages == 0) 739 break; 740 741 for (i = 0; i < pages; i++) { 742 pfn = pagemap_pfn(buf[i]); 743 if (pfn) 744 walk_pfn(index + i, pfn, 1, buf[i]); 745 if (buf[i] & PM_SWAP) 746 walk_swap(index + i, buf[i]); 747 } 748 749 index += pages; 750 count -= pages; 751 } 752 } 753 754 static void walk_task(unsigned long index, unsigned long count) 755 { 756 const unsigned long end = index + count; 757 unsigned long start; 758 int i = 0; 759 760 while (index < end) { 761 762 while (pg_end[i] <= index) 763 if (++i >= nr_vmas) 764 return; 765 if (pg_start[i] >= end) 766 return; 767 768 start = max_t(unsigned long, pg_start[i], index); 769 index = min_t(unsigned long, pg_end[i], end); 770 771 assert(start < index); 772 walk_vma(start, index - start); 773 } 774 } 775 776 static void add_addr_range(unsigned long offset, unsigned long size) 777 { 778 if (nr_addr_ranges >= MAX_ADDR_RANGES) 779 fatal("too many addr ranges\n"); 780 781 opt_offset[nr_addr_ranges] = offset; 782 opt_size[nr_addr_ranges] = min_t(unsigned long, size, ULONG_MAX-offset); 783 nr_addr_ranges++; 784 } 785 786 static void walk_addr_ranges(void) 787 { 788 int i; 789 790 kpageflags_fd = checked_open(opt_kpageflags, O_RDONLY); 791 792 if (!nr_addr_ranges) 793 add_addr_range(0, ULONG_MAX); 794 795 for (i = 0; i < nr_addr_ranges; i++) 796 if (!opt_pid) 797 walk_pfn(opt_offset[i], opt_offset[i], opt_size[i], 0); 798 else 799 walk_task(opt_offset[i], opt_size[i]); 800 801 if (opt_mark_idle) 802 mark_page_idle(0); 803 804 close(kpageflags_fd); 805 } 806 807 808 /* 809 * user interface 810 */ 811 812 static const char *page_flag_type(uint64_t flag) 813 { 814 if (flag & KPF_HACKERS_BITS) 815 return "(r)"; 816 if (flag & KPF_OVERLOADED_BITS) 817 return "(o)"; 818 return " "; 819 } 820 821 static void usage(void) 822 { 823 size_t i, j; 824 825 printf( 826 "page-types [options]\n" 827 " -r|--raw Raw mode, for kernel developers\n" 828 " -d|--describe flags Describe flags\n" 829 " -a|--addr addr-spec Walk a range of pages\n" 830 " -b|--bits bits-spec Walk pages with specified bits\n" 831 " -c|--cgroup path|@inode Walk pages within memory cgroup\n" 832 " -p|--pid pid Walk process address space\n" 833 " -f|--file filename Walk file address space\n" 834 " -i|--mark-idle Mark pages idle\n" 835 " -l|--list Show page details in ranges\n" 836 " -L|--list-each Show page details one by one\n" 837 " -C|--list-cgroup Show cgroup inode for pages\n" 838 " -M|--list-mapcnt Show page map count\n" 839 " -N|--no-summary Don't show summary info\n" 840 " -X|--hwpoison hwpoison pages\n" 841 " -x|--unpoison unpoison pages\n" 842 " -F|--kpageflags filename kpageflags file to parse\n" 843 " -h|--help Show this usage message\n" 844 "flags:\n" 845 " 0x10 bitfield format, e.g.\n" 846 " anon bit-name, e.g.\n" 847 " 0x10,anon comma-separated list, e.g.\n" 848 "addr-spec:\n" 849 " N one page at offset N (unit: pages)\n" 850 " N+M pages range from N to N+M-1\n" 851 " N,M pages range from N to M-1\n" 852 " N, pages range from N to end\n" 853 " ,M pages range from 0 to M-1\n" 854 "bits-spec:\n" 855 " bit1,bit2 (flags & (bit1|bit2)) != 0\n" 856 " bit1,bit2=bit1 (flags & (bit1|bit2)) == bit1\n" 857 " bit1,~bit2 (flags & (bit1|bit2)) == bit1\n" 858 " =bit1,bit2 flags == (bit1|bit2)\n" 859 "bit-names:\n" 860 ); 861 862 for (i = 0, j = 0; i < ARRAY_SIZE(page_flag_names); i++) { 863 if (!page_flag_names[i]) 864 continue; 865 printf("%16s%s", page_flag_names[i] + 2, 866 page_flag_type(1ULL << i)); 867 if (++j > 3) { 868 j = 0; 869 putchar('\n'); 870 } 871 } 872 printf("\n " 873 "(r) raw mode bits (o) overloaded bits\n"); 874 } 875 876 static unsigned long long parse_number(const char *str) 877 { 878 unsigned long long n; 879 880 n = strtoll(str, NULL, 0); 881 882 if (n == 0 && str[0] != '0') 883 fatal("invalid name or number: %s\n", str); 884 885 return n; 886 } 887 888 static void parse_pid(const char *str) 889 { 890 FILE *file; 891 char buf[5000]; 892 893 opt_pid = parse_number(str); 894 895 sprintf(buf, "/proc/%d/pagemap", opt_pid); 896 pagemap_fd = checked_open(buf, O_RDONLY); 897 898 sprintf(buf, "/proc/%d/maps", opt_pid); 899 file = fopen(buf, "r"); 900 if (!file) { 901 perror(buf); 902 exit(EXIT_FAILURE); 903 } 904 905 while (fgets(buf, sizeof(buf), file) != NULL) { 906 unsigned long vm_start; 907 unsigned long vm_end; 908 unsigned long long pgoff; 909 int major, minor; 910 char r, w, x, s; 911 unsigned long ino; 912 int n; 913 914 n = sscanf(buf, "%lx-%lx %c%c%c%c %llx %x:%x %lu", 915 &vm_start, 916 &vm_end, 917 &r, &w, &x, &s, 918 &pgoff, 919 &major, &minor, 920 &ino); 921 if (n < 10) { 922 fprintf(stderr, "unexpected line: %s\n", buf); 923 continue; 924 } 925 pg_start[nr_vmas] = vm_start / page_size; 926 pg_end[nr_vmas] = vm_end / page_size; 927 if (++nr_vmas >= MAX_VMAS) { 928 fprintf(stderr, "too many VMAs\n"); 929 break; 930 } 931 } 932 fclose(file); 933 } 934 935 static void show_file(const char *name, const struct stat *st) 936 { 937 unsigned long long size = st->st_size; 938 char atime[64], mtime[64]; 939 long now = time(NULL); 940 941 printf("%s\tInode: %u\tSize: %llu (%llu pages)\n", 942 name, (unsigned)st->st_ino, 943 size, (size + page_size - 1) / page_size); 944 945 strftime(atime, sizeof(atime), "%c", localtime(&st->st_atime)); 946 strftime(mtime, sizeof(mtime), "%c", localtime(&st->st_mtime)); 947 948 printf("Modify: %s (%ld seconds ago)\nAccess: %s (%ld seconds ago)\n", 949 mtime, now - st->st_mtime, 950 atime, now - st->st_atime); 951 } 952 953 static sigjmp_buf sigbus_jmp; 954 955 static void * volatile sigbus_addr; 956 957 static void sigbus_handler(int sig, siginfo_t *info, void *ucontex) 958 { 959 (void)sig; 960 (void)ucontex; 961 sigbus_addr = info ? info->si_addr : NULL; 962 siglongjmp(sigbus_jmp, 1); 963 } 964 965 static struct sigaction sigbus_action = { 966 .sa_sigaction = sigbus_handler, 967 .sa_flags = SA_SIGINFO, 968 }; 969 970 static void walk_file_range(const char *name, int fd, 971 unsigned long off, unsigned long end) 972 { 973 uint8_t vec[PAGEMAP_BATCH]; 974 uint64_t buf[PAGEMAP_BATCH], flags; 975 uint64_t cgroup = 0; 976 uint64_t mapcnt = 0; 977 unsigned long nr_pages, pfn, i; 978 ssize_t len; 979 void *ptr; 980 int first = 1; 981 982 for (; off < end; off += len) { 983 nr_pages = (end - off + page_size - 1) / page_size; 984 if (nr_pages > PAGEMAP_BATCH) 985 nr_pages = PAGEMAP_BATCH; 986 len = nr_pages * page_size; 987 988 ptr = mmap(NULL, len, PROT_READ, MAP_SHARED, fd, off); 989 if (ptr == MAP_FAILED) 990 fatal("mmap failed: %s", name); 991 992 /* determine cached pages */ 993 if (mincore(ptr, len, vec)) 994 fatal("mincore failed: %s", name); 995 996 /* turn off readahead */ 997 if (madvise(ptr, len, MADV_RANDOM)) 998 fatal("madvise failed: %s", name); 999 1000 if (sigsetjmp(sigbus_jmp, 1)) { 1001 end = off + (sigbus_addr ? sigbus_addr - ptr : 0); 1002 fprintf(stderr, "got sigbus at offset %lld: %s\n", 1003 (long long)end, name); 1004 goto got_sigbus; 1005 } 1006 1007 /* populate ptes */ 1008 for (i = 0; i < nr_pages ; i++) { 1009 if (vec[i] & 1) 1010 (void)*(volatile int *)(ptr + i * page_size); 1011 } 1012 got_sigbus: 1013 1014 /* turn off harvesting reference bits */ 1015 if (madvise(ptr, len, MADV_SEQUENTIAL)) 1016 fatal("madvise failed: %s", name); 1017 1018 if (pagemap_read(buf, (unsigned long)ptr / page_size, 1019 nr_pages) != nr_pages) 1020 fatal("cannot read pagemap"); 1021 1022 munmap(ptr, len); 1023 1024 for (i = 0; i < nr_pages; i++) { 1025 pfn = pagemap_pfn(buf[i]); 1026 if (!pfn) 1027 continue; 1028 if (!kpageflags_read(&flags, pfn, 1)) 1029 continue; 1030 if (!kpagecgroup_read(&cgroup, pfn, 1)) 1031 fatal("kpagecgroup_read failed"); 1032 if (!kpagecount_read(&mapcnt, pfn, 1)) 1033 fatal("kpagecount_read failed"); 1034 if (first && opt_list) { 1035 first = 0; 1036 flush_page_range(); 1037 } 1038 add_page(off / page_size + i, pfn, 1039 flags, cgroup, mapcnt, buf[i]); 1040 } 1041 } 1042 } 1043 1044 static void walk_file(const char *name, const struct stat *st) 1045 { 1046 int i; 1047 int fd; 1048 1049 fd = checked_open(name, O_RDONLY|O_NOATIME|O_NOFOLLOW); 1050 1051 if (!nr_addr_ranges) 1052 add_addr_range(0, st->st_size / page_size); 1053 1054 for (i = 0; i < nr_addr_ranges; i++) 1055 walk_file_range(name, fd, opt_offset[i] * page_size, 1056 (opt_offset[i] + opt_size[i]) * page_size); 1057 1058 close(fd); 1059 } 1060 1061 int walk_tree(const char *name, const struct stat *st, int type, struct FTW *f) 1062 { 1063 (void)f; 1064 switch (type) { 1065 case FTW_F: 1066 if (S_ISREG(st->st_mode)) 1067 walk_file(name, st); 1068 break; 1069 case FTW_DNR: 1070 fprintf(stderr, "cannot read dir: %s\n", name); 1071 break; 1072 } 1073 return 0; 1074 } 1075 1076 struct stat st; 1077 1078 static void walk_page_cache(void) 1079 { 1080 kpageflags_fd = checked_open(opt_kpageflags, O_RDONLY); 1081 pagemap_fd = checked_open("/proc/self/pagemap", O_RDONLY); 1082 sigaction(SIGBUS, &sigbus_action, NULL); 1083 1084 if (stat(opt_file, &st)) 1085 fatal("stat failed: %s\n", opt_file); 1086 1087 if (S_ISREG(st.st_mode)) { 1088 walk_file(opt_file, &st); 1089 } else if (S_ISDIR(st.st_mode)) { 1090 /* do not follow symlinks and mountpoints */ 1091 if (nftw(opt_file, walk_tree, 64, FTW_MOUNT | FTW_PHYS) < 0) 1092 fatal("nftw failed: %s\n", opt_file); 1093 } else 1094 fatal("unhandled file type: %s\n", opt_file); 1095 1096 close(kpageflags_fd); 1097 close(pagemap_fd); 1098 signal(SIGBUS, SIG_DFL); 1099 } 1100 1101 static void parse_file(const char *name) 1102 { 1103 opt_file = name; 1104 } 1105 1106 static void parse_cgroup(const char *path) 1107 { 1108 if (path[0] == '@') { 1109 opt_cgroup = parse_number(path + 1); 1110 return; 1111 } 1112 1113 struct stat st; 1114 1115 if (stat(path, &st)) 1116 fatal("stat failed: %s: %m\n", path); 1117 1118 if (!S_ISDIR(st.st_mode)) 1119 fatal("cgroup supposed to be a directory: %s\n", path); 1120 1121 opt_cgroup = st.st_ino; 1122 } 1123 1124 static void parse_addr_range(const char *optarg) 1125 { 1126 unsigned long offset; 1127 unsigned long size; 1128 char *p; 1129 1130 p = strchr(optarg, ','); 1131 if (!p) 1132 p = strchr(optarg, '+'); 1133 1134 if (p == optarg) { 1135 offset = 0; 1136 size = parse_number(p + 1); 1137 } else if (p) { 1138 offset = parse_number(optarg); 1139 if (p[1] == '\0') 1140 size = ULONG_MAX; 1141 else { 1142 size = parse_number(p + 1); 1143 if (*p == ',') { 1144 if (size < offset) 1145 fatal("invalid range: %lu,%lu\n", 1146 offset, size); 1147 size -= offset; 1148 } 1149 } 1150 } else { 1151 offset = parse_number(optarg); 1152 size = 1; 1153 } 1154 1155 add_addr_range(offset, size); 1156 } 1157 1158 static void add_bits_filter(uint64_t mask, uint64_t bits) 1159 { 1160 if (nr_bit_filters >= MAX_BIT_FILTERS) 1161 fatal("too much bit filters\n"); 1162 1163 opt_mask[nr_bit_filters] = mask; 1164 opt_bits[nr_bit_filters] = bits; 1165 nr_bit_filters++; 1166 } 1167 1168 static uint64_t parse_flag_name(const char *str, int len) 1169 { 1170 size_t i; 1171 1172 if (!*str || !len) 1173 return 0; 1174 1175 if (len <= 8 && !strncmp(str, "compound", len)) 1176 return BITS_COMPOUND; 1177 1178 for (i = 0; i < ARRAY_SIZE(page_flag_names); i++) { 1179 if (!page_flag_names[i]) 1180 continue; 1181 if (!strncmp(str, page_flag_names[i] + 2, len)) 1182 return 1ULL << i; 1183 } 1184 1185 return parse_number(str); 1186 } 1187 1188 static uint64_t parse_flag_names(const char *str, int all) 1189 { 1190 const char *p = str; 1191 uint64_t flags = 0; 1192 1193 while (1) { 1194 if (*p == ',' || *p == '=' || *p == '\0') { 1195 if ((*str != '~') || (*str == '~' && all && *++str)) 1196 flags |= parse_flag_name(str, p - str); 1197 if (*p != ',') 1198 break; 1199 str = p + 1; 1200 } 1201 p++; 1202 } 1203 1204 return flags; 1205 } 1206 1207 static void parse_bits_mask(const char *optarg) 1208 { 1209 uint64_t mask; 1210 uint64_t bits; 1211 const char *p; 1212 1213 p = strchr(optarg, '='); 1214 if (p == optarg) { 1215 mask = KPF_ALL_BITS; 1216 bits = parse_flag_names(p + 1, 0); 1217 } else if (p) { 1218 mask = parse_flag_names(optarg, 0); 1219 bits = parse_flag_names(p + 1, 0); 1220 } else if (strchr(optarg, '~')) { 1221 mask = parse_flag_names(optarg, 1); 1222 bits = parse_flag_names(optarg, 0); 1223 } else { 1224 mask = parse_flag_names(optarg, 0); 1225 bits = KPF_ALL_BITS; 1226 } 1227 1228 add_bits_filter(mask, bits); 1229 } 1230 1231 static void parse_kpageflags(const char *name) 1232 { 1233 opt_kpageflags = name; 1234 } 1235 1236 static void describe_flags(const char *optarg) 1237 { 1238 uint64_t flags = parse_flag_names(optarg, 0); 1239 1240 printf("0x%016llx\t%s\t%s\n", 1241 (unsigned long long)flags, 1242 page_flag_name(flags), 1243 page_flag_longname(flags)); 1244 } 1245 1246 static const struct option opts[] = { 1247 { "raw" , 0, NULL, 'r' }, 1248 { "pid" , 1, NULL, 'p' }, 1249 { "file" , 1, NULL, 'f' }, 1250 { "addr" , 1, NULL, 'a' }, 1251 { "bits" , 1, NULL, 'b' }, 1252 { "cgroup" , 1, NULL, 'c' }, 1253 { "describe" , 1, NULL, 'd' }, 1254 { "mark-idle" , 0, NULL, 'i' }, 1255 { "list" , 0, NULL, 'l' }, 1256 { "list-each" , 0, NULL, 'L' }, 1257 { "list-cgroup", 0, NULL, 'C' }, 1258 { "list-mapcnt", 0, NULL, 'M' }, 1259 { "no-summary", 0, NULL, 'N' }, 1260 { "hwpoison" , 0, NULL, 'X' }, 1261 { "unpoison" , 0, NULL, 'x' }, 1262 { "kpageflags", 1, NULL, 'F' }, 1263 { "help" , 0, NULL, 'h' }, 1264 { NULL , 0, NULL, 0 } 1265 }; 1266 1267 int main(int argc, char *argv[]) 1268 { 1269 int c; 1270 1271 page_size = getpagesize(); 1272 1273 while ((c = getopt_long(argc, argv, 1274 "rp:f:a:b:d:c:CilLMNXxF:h", 1275 opts, NULL)) != -1) { 1276 switch (c) { 1277 case 'r': 1278 opt_raw = 1; 1279 break; 1280 case 'p': 1281 parse_pid(optarg); 1282 break; 1283 case 'f': 1284 parse_file(optarg); 1285 break; 1286 case 'a': 1287 parse_addr_range(optarg); 1288 break; 1289 case 'b': 1290 parse_bits_mask(optarg); 1291 break; 1292 case 'c': 1293 parse_cgroup(optarg); 1294 break; 1295 case 'C': 1296 opt_list_cgroup = 1; 1297 break; 1298 case 'd': 1299 describe_flags(optarg); 1300 exit(0); 1301 case 'i': 1302 opt_mark_idle = 1; 1303 break; 1304 case 'l': 1305 opt_list = 1; 1306 break; 1307 case 'L': 1308 opt_list = 2; 1309 break; 1310 case 'M': 1311 opt_list_mapcnt = 1; 1312 break; 1313 case 'N': 1314 opt_no_summary = 1; 1315 break; 1316 case 'X': 1317 opt_hwpoison = 1; 1318 prepare_hwpoison_fd(); 1319 break; 1320 case 'x': 1321 opt_unpoison = 1; 1322 prepare_hwpoison_fd(); 1323 break; 1324 case 'F': 1325 parse_kpageflags(optarg); 1326 break; 1327 case 'h': 1328 usage(); 1329 exit(0); 1330 default: 1331 usage(); 1332 exit(1); 1333 } 1334 } 1335 1336 if (!opt_kpageflags) 1337 opt_kpageflags = PROC_KPAGEFLAGS; 1338 1339 if (opt_cgroup || opt_list_cgroup) 1340 kpagecgroup_fd = checked_open(PROC_KPAGECGROUP, O_RDONLY); 1341 1342 if (opt_list && opt_list_mapcnt) 1343 kpagecount_fd = checked_open(PROC_KPAGECOUNT, O_RDONLY); 1344 1345 if (opt_mark_idle) 1346 page_idle_fd = checked_open(SYS_KERNEL_MM_PAGE_IDLE, O_RDWR); 1347 1348 if (opt_list && opt_pid) 1349 printf("voffset\t"); 1350 if (opt_list && opt_file) 1351 printf("foffset\t"); 1352 if (opt_list && opt_list_cgroup) 1353 printf("cgroup\t"); 1354 if (opt_list && opt_list_mapcnt) 1355 printf("map-cnt\t"); 1356 1357 if (opt_list == 1) 1358 printf("offset\tlen\tflags\n"); 1359 if (opt_list == 2) 1360 printf("offset\tflags\n"); 1361 1362 if (opt_file) 1363 walk_page_cache(); 1364 else 1365 walk_addr_ranges(); 1366 1367 if (opt_list == 1) 1368 flush_page_range(); 1369 1370 if (opt_no_summary) 1371 return 0; 1372 1373 if (opt_list) 1374 printf("\n\n"); 1375 1376 if (opt_file) { 1377 show_file(opt_file, &st); 1378 printf("\n"); 1379 } 1380 1381 show_summary(); 1382 1383 if (opt_list_mapcnt) 1384 close(kpagecount_fd); 1385 1386 if (page_idle_fd >= 0) 1387 close(page_idle_fd); 1388 1389 return 0; 1390 } 1391