1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/debugfs.h> 3 #include <linux/mm.h> 4 #include <linux/slab.h> 5 #include <linux/uaccess.h> 6 #include <linux/memblock.h> 7 #include <linux/stacktrace.h> 8 #include <linux/page_owner.h> 9 #include <linux/jump_label.h> 10 #include <linux/migrate.h> 11 #include <linux/stackdepot.h> 12 #include <linux/seq_file.h> 13 #include <linux/memcontrol.h> 14 #include <linux/sched/clock.h> 15 16 #include "page_alloc.h" 17 18 /* 19 * TODO: teach PAGE_OWNER_STACK_DEPTH (__dump_page_owner and save_stack) 20 * to use off stack temporal storage 21 */ 22 #define PAGE_OWNER_STACK_DEPTH (16) 23 24 struct page_owner { 25 unsigned short order; 26 short last_migrate_reason; 27 gfp_t gfp_mask; 28 depot_stack_handle_t handle; 29 depot_stack_handle_t free_handle; 30 u64 ts_nsec; 31 u64 free_ts_nsec; 32 char comm[TASK_COMM_LEN]; 33 pid_t pid; 34 pid_t tgid; 35 pid_t free_pid; 36 pid_t free_tgid; 37 }; 38 39 struct stack { 40 struct stack_record *stack_record; 41 struct stack *next; 42 }; 43 static struct stack dummy_stack; 44 static struct stack failure_stack; 45 static struct stack *stack_list; 46 static DEFINE_SPINLOCK(stack_list_lock); 47 48 #define STACK_PRINT_FLAG_STACK 0x1 49 #define STACK_PRINT_FLAG_PAGES 0x2 50 #define STACK_PRINT_FLAG_HANDLE 0x4 51 52 struct stack_print_ctx { 53 struct stack *stack; 54 u8 flags; 55 }; 56 57 enum page_owner_print_mode { 58 PAGE_OWNER_PRINT_STACK, 59 PAGE_OWNER_PRINT_HANDLE, 60 PAGE_OWNER_PRINT_STACK_HANDLE, 61 }; 62 63 static const char * const page_owner_print_mode_strings[] = { 64 [PAGE_OWNER_PRINT_STACK] = "stack", 65 [PAGE_OWNER_PRINT_HANDLE] = "handle", 66 [PAGE_OWNER_PRINT_STACK_HANDLE] = "stack_handle", 67 }; 68 69 struct page_owner_filter_state { 70 enum page_owner_print_mode print_mode; 71 nodemask_t nid_filter; 72 bool nid_filter_enabled; 73 }; 74 75 static bool page_owner_enabled __initdata; 76 DEFINE_STATIC_KEY_FALSE(page_owner_inited); 77 78 static depot_stack_handle_t dummy_handle; 79 static depot_stack_handle_t failure_handle; 80 static depot_stack_handle_t early_handle; 81 82 static void init_early_allocated_pages(void); 83 84 static inline void set_current_in_page_owner(void) 85 { 86 /* 87 * Avoid recursion. 88 * 89 * We might need to allocate more memory from page_owner code, so make 90 * sure to signal it in order to avoid recursion. 91 */ 92 current->in_page_owner = 1; 93 } 94 95 static inline void unset_current_in_page_owner(void) 96 { 97 current->in_page_owner = 0; 98 } 99 100 static int __init early_page_owner_param(char *buf) 101 { 102 int ret = kstrtobool(buf, &page_owner_enabled); 103 104 if (page_owner_enabled) 105 stack_depot_request_early_init(); 106 107 return ret; 108 } 109 early_param("page_owner", early_page_owner_param); 110 111 static __init bool need_page_owner(void) 112 { 113 return page_owner_enabled; 114 } 115 116 static __always_inline depot_stack_handle_t create_dummy_stack(void) 117 { 118 unsigned long entries[4]; 119 unsigned int nr_entries; 120 121 nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 0); 122 return stack_depot_save(entries, nr_entries, GFP_KERNEL); 123 } 124 125 static noinline void register_dummy_stack(void) 126 { 127 dummy_handle = create_dummy_stack(); 128 } 129 130 static noinline void register_failure_stack(void) 131 { 132 failure_handle = create_dummy_stack(); 133 } 134 135 static noinline void register_early_stack(void) 136 { 137 early_handle = create_dummy_stack(); 138 } 139 140 static __init void init_page_owner(void) 141 { 142 if (!page_owner_enabled) 143 return; 144 145 register_dummy_stack(); 146 register_failure_stack(); 147 register_early_stack(); 148 init_early_allocated_pages(); 149 /* Initialize dummy and failure stacks and link them to stack_list */ 150 dummy_stack.stack_record = __stack_depot_get_stack_record(dummy_handle); 151 failure_stack.stack_record = __stack_depot_get_stack_record(failure_handle); 152 if (dummy_stack.stack_record) 153 refcount_set(&dummy_stack.stack_record->count, 1); 154 if (failure_stack.stack_record) 155 refcount_set(&failure_stack.stack_record->count, 1); 156 dummy_stack.next = &failure_stack; 157 stack_list = &dummy_stack; 158 static_branch_enable(&page_owner_inited); 159 } 160 161 struct page_ext_operations page_owner_ops = { 162 .size = sizeof(struct page_owner), 163 .need = need_page_owner, 164 .init = init_page_owner, 165 .need_shared_flags = true, 166 }; 167 168 static inline struct page_owner *get_page_owner(struct page_ext *page_ext) 169 { 170 return page_ext_data(page_ext, &page_owner_ops); 171 } 172 173 static noinline depot_stack_handle_t save_stack(gfp_t flags) 174 { 175 unsigned long entries[PAGE_OWNER_STACK_DEPTH]; 176 depot_stack_handle_t handle; 177 unsigned int nr_entries; 178 179 if (current->in_page_owner) 180 return dummy_handle; 181 182 set_current_in_page_owner(); 183 nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 2); 184 handle = stack_depot_save(entries, nr_entries, flags); 185 if (!handle) 186 handle = failure_handle; 187 unset_current_in_page_owner(); 188 189 return handle; 190 } 191 192 static void add_stack_record_to_list(struct stack_record *stack_record, 193 gfp_t gfp_mask) 194 { 195 unsigned long flags; 196 struct stack *stack; 197 198 if (!gfpflags_allow_spinning(gfp_mask)) 199 return; 200 201 set_current_in_page_owner(); 202 stack = kmalloc_obj(*stack, gfp_nested_mask(gfp_mask)); 203 if (!stack) { 204 unset_current_in_page_owner(); 205 return; 206 } 207 unset_current_in_page_owner(); 208 209 stack->stack_record = stack_record; 210 stack->next = NULL; 211 212 spin_lock_irqsave(&stack_list_lock, flags); 213 stack->next = stack_list; 214 /* 215 * This pairs with smp_load_acquire() from function 216 * stack_start(). This guarantees that stack_start() 217 * will see an updated stack_list before starting to 218 * traverse the list. 219 */ 220 smp_store_release(&stack_list, stack); 221 spin_unlock_irqrestore(&stack_list_lock, flags); 222 } 223 224 static void inc_stack_record_count(depot_stack_handle_t handle, gfp_t gfp_mask, 225 int nr_base_pages) 226 { 227 struct stack_record *stack_record = __stack_depot_get_stack_record(handle); 228 229 if (!stack_record) 230 return; 231 232 /* 233 * New stack_record's that do not use STACK_DEPOT_FLAG_GET start 234 * with REFCOUNT_SATURATED to catch spurious increments of their 235 * refcount. 236 * Since we do not use STACK_DEPOT_FLAG_GET API, let us 237 * set a refcount of 1 ourselves. 238 */ 239 if (refcount_read(&stack_record->count) == REFCOUNT_SATURATED) { 240 int old = REFCOUNT_SATURATED; 241 242 if (atomic_try_cmpxchg_relaxed(&stack_record->count.refs, &old, 1)) 243 /* Add the new stack_record to our list */ 244 add_stack_record_to_list(stack_record, gfp_mask); 245 } 246 refcount_add(nr_base_pages, &stack_record->count); 247 } 248 249 static void dec_stack_record_count(depot_stack_handle_t handle, 250 int nr_base_pages) 251 { 252 struct stack_record *stack_record = __stack_depot_get_stack_record(handle); 253 254 if (!stack_record) 255 return; 256 257 if (refcount_sub_and_test(nr_base_pages, &stack_record->count)) 258 pr_warn("%s: refcount went to 0 for %u handle\n", __func__, 259 handle); 260 } 261 262 static inline void __update_page_owner_handle(struct page *page, 263 depot_stack_handle_t handle, 264 unsigned short order, 265 gfp_t gfp_mask, 266 short last_migrate_reason, u64 ts_nsec, 267 pid_t pid, pid_t tgid, char *comm) 268 { 269 struct page_ext_iter iter; 270 struct page_ext *page_ext; 271 struct page_owner *page_owner; 272 273 rcu_read_lock(); 274 for_each_page_ext(page, 1 << order, page_ext, iter) { 275 page_owner = get_page_owner(page_ext); 276 page_owner->handle = handle; 277 page_owner->order = order; 278 page_owner->gfp_mask = gfp_mask; 279 page_owner->last_migrate_reason = last_migrate_reason; 280 page_owner->pid = pid; 281 page_owner->tgid = tgid; 282 page_owner->ts_nsec = ts_nsec; 283 strscpy(page_owner->comm, comm, 284 sizeof(page_owner->comm)); 285 __set_bit(PAGE_EXT_OWNER, &page_ext->flags); 286 __set_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags); 287 } 288 rcu_read_unlock(); 289 } 290 291 static inline void __update_page_owner_free_handle(struct page *page, 292 depot_stack_handle_t handle, 293 unsigned short order, 294 pid_t pid, pid_t tgid, 295 u64 free_ts_nsec) 296 { 297 struct page_ext_iter iter; 298 struct page_ext *page_ext; 299 struct page_owner *page_owner; 300 301 rcu_read_lock(); 302 for_each_page_ext(page, 1 << order, page_ext, iter) { 303 page_owner = get_page_owner(page_ext); 304 /* Only __reset_page_owner() wants to clear the bit */ 305 if (handle) { 306 __clear_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags); 307 page_owner->free_handle = handle; 308 } 309 page_owner->free_ts_nsec = free_ts_nsec; 310 page_owner->free_pid = current->pid; 311 page_owner->free_tgid = current->tgid; 312 } 313 rcu_read_unlock(); 314 } 315 316 void __reset_page_owner(struct page *page, unsigned short order) 317 { 318 struct page_ext *page_ext; 319 depot_stack_handle_t handle; 320 depot_stack_handle_t alloc_handle; 321 struct page_owner *page_owner; 322 u64 free_ts_nsec = local_clock(); 323 324 page_ext = page_ext_get(page); 325 if (unlikely(!page_ext)) 326 return; 327 328 page_owner = get_page_owner(page_ext); 329 alloc_handle = page_owner->handle; 330 page_ext_put(page_ext); 331 332 /* 333 * Do not specify GFP_NOWAIT to make gfpflags_allow_spinning() == false 334 * to prevent issues in stack_depot_save(). 335 * This is similar to alloc_pages_nolock() gfp flags, but only used 336 * to signal stack_depot to avoid spin_locks. 337 */ 338 handle = save_stack(__GFP_NOWARN); 339 __update_page_owner_free_handle(page, handle, order, current->pid, 340 current->tgid, free_ts_nsec); 341 342 if (alloc_handle != early_handle) 343 /* 344 * early_handle is being set as a handle for all those 345 * early allocated pages. See init_pages_in_zone(). 346 * Since their refcount is not being incremented because 347 * the machinery is not ready yet, we cannot decrement 348 * their refcount either. 349 */ 350 dec_stack_record_count(alloc_handle, 1 << order); 351 } 352 353 noinline void __set_page_owner(struct page *page, unsigned short order, 354 gfp_t gfp_mask) 355 { 356 u64 ts_nsec = local_clock(); 357 depot_stack_handle_t handle; 358 359 handle = save_stack(gfp_mask); 360 __update_page_owner_handle(page, handle, order, gfp_mask, MR_NEVER, 361 ts_nsec, current->pid, current->tgid, 362 current->comm); 363 inc_stack_record_count(handle, gfp_mask, 1 << order); 364 } 365 366 void __folio_set_owner_migrate_reason(struct folio *folio, enum migrate_reason reason) 367 { 368 struct page_ext *page_ext = page_ext_get(&folio->page); 369 struct page_owner *page_owner; 370 371 if (unlikely(!page_ext)) 372 return; 373 374 page_owner = get_page_owner(page_ext); 375 page_owner->last_migrate_reason = reason; 376 page_ext_put(page_ext); 377 } 378 379 void __split_page_owner(struct page *page, int old_order, int new_order) 380 { 381 struct page_ext_iter iter; 382 struct page_ext *page_ext; 383 struct page_owner *page_owner; 384 385 rcu_read_lock(); 386 for_each_page_ext(page, 1 << old_order, page_ext, iter) { 387 page_owner = get_page_owner(page_ext); 388 page_owner->order = new_order; 389 } 390 rcu_read_unlock(); 391 } 392 393 void __folio_copy_owner(struct folio *newfolio, struct folio *old) 394 { 395 struct page_ext *page_ext; 396 struct page_ext_iter iter; 397 struct page_owner *old_page_owner; 398 struct page_owner *new_page_owner; 399 depot_stack_handle_t migrate_handle; 400 401 page_ext = page_ext_get(&old->page); 402 if (unlikely(!page_ext)) 403 return; 404 405 old_page_owner = get_page_owner(page_ext); 406 page_ext_put(page_ext); 407 408 page_ext = page_ext_get(&newfolio->page); 409 if (unlikely(!page_ext)) 410 return; 411 412 new_page_owner = get_page_owner(page_ext); 413 page_ext_put(page_ext); 414 415 migrate_handle = new_page_owner->handle; 416 __update_page_owner_handle(&newfolio->page, old_page_owner->handle, 417 old_page_owner->order, old_page_owner->gfp_mask, 418 old_page_owner->last_migrate_reason, 419 old_page_owner->ts_nsec, old_page_owner->pid, 420 old_page_owner->tgid, old_page_owner->comm); 421 /* 422 * Do not proactively clear PAGE_EXT_OWNER{_ALLOCATED} bits as the folio 423 * will be freed after migration. Keep them until then as they may be 424 * useful. 425 */ 426 __update_page_owner_free_handle(&newfolio->page, 0, old_page_owner->order, 427 old_page_owner->free_pid, 428 old_page_owner->free_tgid, 429 old_page_owner->free_ts_nsec); 430 /* 431 * We linked the original stack to the new folio, we need to do the same 432 * for the new one and the old folio otherwise there will be an imbalance 433 * when subtracting those pages from the stack. 434 */ 435 rcu_read_lock(); 436 for_each_page_ext(&old->page, 1 << new_page_owner->order, page_ext, iter) { 437 old_page_owner = get_page_owner(page_ext); 438 old_page_owner->handle = migrate_handle; 439 } 440 rcu_read_unlock(); 441 } 442 443 /* 444 * Check if a page is a buddy page and advance @pfn past the entire buddy block. 445 * This safely reads the buddy order without the zone lock, which may cause us 446 * to skip less than the full buddy block, but that is acceptable for page owner 447 * iteration purposes. 448 * 449 * The lockless read of buddy_order_unsafe() can also return a garbage order if 450 * the page is concurrently allocated and PageBuddy is cleared between the check 451 * and the read. Clamp the advance at the next MAX_ORDER_NR_PAGES boundary so 452 * that a bogus order cannot carry @pfn into an unvalidated memory section, 453 * which would break callers that rely on boundary-aligned pfn_valid() checks. 454 * 455 * Return: true if the page was skipped (caller should continue its loop), 456 * false if the page is not a buddy page and should be processed normally. 457 */ 458 static inline bool skip_buddy_pages(unsigned long *pfn, struct page *page) 459 { 460 unsigned long order; 461 462 if (!PageBuddy(page)) 463 return false; 464 465 order = buddy_order_unsafe(page); 466 if (order <= MAX_PAGE_ORDER) { 467 unsigned long new_pfn = *pfn + (1UL << order); 468 unsigned long boundary = ALIGN(*pfn + 1, MAX_ORDER_NR_PAGES); 469 470 *pfn = min(new_pfn, boundary) - 1; 471 } 472 473 return true; 474 } 475 476 void pagetypeinfo_showmixedcount_print(struct seq_file *m, 477 pg_data_t *pgdat, struct zone *zone) 478 { 479 struct page *page; 480 struct page_ext *page_ext; 481 struct page_owner *page_owner; 482 unsigned long pfn, block_end_pfn; 483 unsigned long end_pfn = zone_end_pfn(zone); 484 unsigned long count[MIGRATE_TYPES] = { 0, }; 485 int pageblock_mt, page_mt; 486 int i; 487 488 /* Scan block by block. First and last block may be incomplete */ 489 pfn = zone->zone_start_pfn; 490 491 /* 492 * Walk the zone in pageblock_nr_pages steps. If a page block spans 493 * a zone boundary, it will be double counted between zones. This does 494 * not matter as the mixed block count will still be correct 495 */ 496 for (; pfn < end_pfn; ) { 497 page = pfn_to_online_page(pfn); 498 if (!page) { 499 pfn = ALIGN(pfn + 1, MAX_ORDER_NR_PAGES); 500 continue; 501 } 502 503 block_end_pfn = pageblock_end_pfn(pfn); 504 block_end_pfn = min(block_end_pfn, end_pfn); 505 506 pageblock_mt = get_pageblock_migratetype(page); 507 508 for (; pfn < block_end_pfn; pfn++) { 509 /* The pageblock is online, no need to recheck. */ 510 page = pfn_to_page(pfn); 511 512 if (page_zone(page) != zone) 513 continue; 514 515 if (skip_buddy_pages(&pfn, page)) 516 continue; 517 518 if (PageReserved(page)) 519 continue; 520 521 page_ext = page_ext_get(page); 522 if (unlikely(!page_ext)) 523 continue; 524 525 if (!test_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags)) 526 goto ext_put_continue; 527 528 page_owner = get_page_owner(page_ext); 529 page_mt = gfp_migratetype(page_owner->gfp_mask); 530 if (pageblock_mt != page_mt) { 531 if (is_migrate_cma(pageblock_mt)) 532 count[MIGRATE_MOVABLE]++; 533 else 534 count[pageblock_mt]++; 535 536 pfn = block_end_pfn; 537 page_ext_put(page_ext); 538 break; 539 } 540 pfn += (1UL << page_owner->order) - 1; 541 ext_put_continue: 542 page_ext_put(page_ext); 543 } 544 } 545 546 /* Print counts */ 547 seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name); 548 for (i = 0; i < MIGRATE_TYPES; i++) 549 seq_printf(m, "%12lu ", count[i]); 550 seq_putc(m, '\n'); 551 } 552 553 #ifdef CONFIG_MEMCG 554 /* 555 * Looking for memcg information and print it out 556 */ 557 static inline int print_page_owner_memcg(char *kbuf, size_t count, int ret, 558 struct page *page) 559 { 560 unsigned long memcg_data; 561 struct obj_cgroup *objcg; 562 struct mem_cgroup *memcg; 563 bool online; 564 char name[80]; 565 566 rcu_read_lock(); 567 memcg_data = READ_ONCE(page->memcg_data); 568 if (!memcg_data || PageTail(page)) 569 goto out_unlock; 570 571 if (memcg_data & MEMCG_DATA_OBJEXTS) { 572 ret += scnprintf(kbuf + ret, count - ret, 573 "Slab cache page\n"); 574 goto out_unlock; 575 } 576 577 objcg = (void *)(memcg_data & ~OBJEXTS_FLAGS_MASK); 578 memcg = objcg ? obj_cgroup_memcg(objcg) : NULL; 579 if (!memcg) 580 goto out_unlock; 581 582 online = css_is_online(&memcg->css); 583 cgroup_name(memcg->css.cgroup, name, sizeof(name)); 584 ret += scnprintf(kbuf + ret, count - ret, 585 "Charged %sto %smemcg %s\n", 586 (memcg_data & MEMCG_DATA_KMEM) ? "(via objcg) " : "", 587 online ? "" : "offline ", 588 name); 589 out_unlock: 590 rcu_read_unlock(); 591 592 return ret; 593 } 594 #else 595 static inline int print_page_owner_memcg(char *kbuf, size_t count, int ret, 596 struct page *page) 597 { 598 return ret; 599 } 600 #endif 601 602 static ssize_t 603 print_page_owner(char __user *buf, size_t count, unsigned long pfn, 604 struct page *page, struct page_owner *page_owner, 605 depot_stack_handle_t handle, 606 struct page_owner_filter_state *state) 607 { 608 int ret, pageblock_mt, page_mt; 609 char *kbuf; 610 enum page_owner_print_mode print_mode; 611 612 count = min_t(size_t, count, PAGE_SIZE); 613 kbuf = kmalloc(count, GFP_KERNEL); 614 if (!kbuf) 615 return -ENOMEM; 616 617 print_mode = state->print_mode; 618 619 ret = scnprintf(kbuf, count, 620 "Page allocated via order %u, mask %#x(%pGg), pid %d, tgid %d (%s), ts %llu ns\n", 621 page_owner->order, page_owner->gfp_mask, 622 &page_owner->gfp_mask, page_owner->pid, 623 page_owner->tgid, page_owner->comm, 624 page_owner->ts_nsec); 625 626 /* Print information relevant to grouping pages by mobility */ 627 pageblock_mt = get_pageblock_migratetype(page); 628 page_mt = gfp_migratetype(page_owner->gfp_mask); 629 ret += scnprintf(kbuf + ret, count - ret, 630 "PFN 0x%lx type %s Block %lu type %s Flags %pGp\n", 631 pfn, 632 migratetype_names[page_mt], 633 pfn >> pageblock_order, 634 migratetype_names[pageblock_mt], 635 &page->flags.f); 636 637 if (print_mode != PAGE_OWNER_PRINT_HANDLE) { 638 ret += stack_depot_snprint(handle, kbuf + ret, count - ret, 0); 639 if (ret >= count) 640 goto err; 641 } 642 643 if (print_mode != PAGE_OWNER_PRINT_STACK) { 644 ret += scnprintf(kbuf + ret, count - ret, "handle: %u\n", 645 handle); 646 if (ret >= count) 647 goto err; 648 } 649 650 if (page_owner->last_migrate_reason != MR_NEVER) { 651 ret += scnprintf(kbuf + ret, count - ret, 652 "Page has been migrated, last migrate reason: %s\n", 653 migrate_reason_names[page_owner->last_migrate_reason]); 654 } 655 656 ret = print_page_owner_memcg(kbuf, count, ret, page); 657 658 ret += snprintf(kbuf + ret, count - ret, "\n"); 659 if (ret >= count) 660 goto err; 661 662 if (copy_to_user(buf, kbuf, ret)) 663 ret = -EFAULT; 664 665 kfree(kbuf); 666 return ret; 667 668 err: 669 kfree(kbuf); 670 return -ENOMEM; 671 } 672 673 void __dump_page_owner(const struct page *page) 674 { 675 struct page_ext *page_ext = page_ext_get((void *)page); 676 struct page_owner *page_owner; 677 depot_stack_handle_t handle; 678 gfp_t gfp_mask; 679 int mt; 680 681 if (unlikely(!page_ext)) { 682 pr_alert("There is not page extension available.\n"); 683 return; 684 } 685 686 page_owner = get_page_owner(page_ext); 687 gfp_mask = page_owner->gfp_mask; 688 mt = gfp_migratetype(gfp_mask); 689 690 if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags)) { 691 pr_alert("page_owner info is not present (never set?)\n"); 692 page_ext_put(page_ext); 693 return; 694 } 695 696 if (test_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags)) 697 pr_alert("page_owner tracks the page as allocated\n"); 698 else 699 pr_alert("page_owner tracks the page as freed\n"); 700 701 pr_alert("page last allocated via order %u, migratetype %s, gfp_mask %#x(%pGg), pid %d, tgid %d (%s), ts %llu\n", 702 page_owner->order, migratetype_names[mt], gfp_mask, &gfp_mask, 703 page_owner->pid, page_owner->tgid, page_owner->comm, 704 page_owner->ts_nsec); 705 706 handle = READ_ONCE(page_owner->handle); 707 if (!handle) 708 pr_alert("page_owner allocation stack trace missing\n"); 709 else 710 stack_depot_print(handle); 711 712 handle = READ_ONCE(page_owner->free_handle); 713 if (!handle) { 714 pr_alert("page_owner free stack trace missing\n"); 715 } else { 716 pr_alert("page last free pid %d tgid %d ts %llu stack trace:\n", 717 page_owner->free_pid, page_owner->free_tgid, 718 page_owner->free_ts_nsec); 719 stack_depot_print(handle); 720 } 721 722 if (page_owner->last_migrate_reason != MR_NEVER) 723 pr_alert("page has been migrated, last migrate reason: %s\n", 724 migrate_reason_names[page_owner->last_migrate_reason]); 725 page_ext_put(page_ext); 726 } 727 728 static ssize_t 729 read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos) 730 { 731 unsigned long pfn; 732 struct page *page; 733 struct page_ext *page_ext; 734 struct page_owner *page_owner; 735 depot_stack_handle_t handle; 736 struct page_owner_filter_state *state = file->private_data; 737 738 if (!static_branch_unlikely(&page_owner_inited)) 739 return -EINVAL; 740 741 page = NULL; 742 if (*ppos == 0) 743 pfn = min_low_pfn; 744 else 745 pfn = *ppos; 746 /* Find a valid PFN or the start of a MAX_ORDER_NR_PAGES area */ 747 while (!pfn_valid(pfn) && (pfn & (MAX_ORDER_NR_PAGES - 1)) != 0) 748 pfn++; 749 750 /* Find an allocated page */ 751 for (; pfn < max_pfn; pfn++) { 752 /* 753 * This temporary page_owner is required so 754 * that we can avoid the context switches while holding 755 * the rcu lock and copying the page owner information to 756 * user through copy_to_user() or GFP_KERNEL allocations. 757 */ 758 struct page_owner page_owner_tmp; 759 760 /* 761 * If the new page is in a new MAX_ORDER_NR_PAGES area, 762 * validate the area as existing, skip it if not 763 */ 764 if ((pfn & (MAX_ORDER_NR_PAGES - 1)) == 0 && !pfn_valid(pfn)) { 765 pfn += MAX_ORDER_NR_PAGES - 1; 766 continue; 767 } 768 769 page = pfn_to_page(pfn); 770 if (skip_buddy_pages(&pfn, page)) 771 continue; 772 773 page_ext = page_ext_get(page); 774 if (unlikely(!page_ext)) 775 continue; 776 777 /* 778 * Some pages could be missed by concurrent allocation or free, 779 * because we don't hold the zone lock. 780 */ 781 if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags)) 782 goto ext_put_continue; 783 784 /* 785 * Although we do have the info about past allocation of free 786 * pages, it's not relevant for current memory usage. 787 */ 788 if (!test_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags)) 789 goto ext_put_continue; 790 791 page_owner = get_page_owner(page_ext); 792 793 /* 794 * Don't print "tail" pages of high-order allocations as that 795 * would inflate the stats. 796 */ 797 if (!IS_ALIGNED(pfn, 1 << page_owner->order)) 798 goto ext_put_continue; 799 800 /* 801 * Access to page_ext->handle isn't synchronous so we should 802 * be careful to access it. 803 */ 804 handle = READ_ONCE(page_owner->handle); 805 if (!handle) 806 goto ext_put_continue; 807 808 if (state->nid_filter_enabled) { 809 int nid; 810 memdesc_flags_t page_flags = READ_ONCE(page->flags); 811 812 /* 813 * Bypass PF_POISONED_CHECK() in page_to_nid() to avoid 814 * VM_BUG_ON when accessing poisoned pages. 815 */ 816 if (page_flags.f == PAGE_POISON_PATTERN) 817 goto ext_put_continue; 818 nid = memdesc_nid(&page_flags); 819 if (!node_isset(nid, state->nid_filter)) 820 goto ext_put_continue; 821 } 822 823 /* Record the next PFN to read in the file offset */ 824 *ppos = pfn + 1; 825 826 page_owner_tmp = *page_owner; 827 page_ext_put(page_ext); 828 return print_page_owner(buf, count, pfn, page, 829 &page_owner_tmp, handle, state); 830 ext_put_continue: 831 page_ext_put(page_ext); 832 cond_resched(); 833 } 834 835 return 0; 836 } 837 838 static loff_t lseek_page_owner(struct file *file, loff_t offset, int orig) 839 { 840 switch (orig) { 841 case SEEK_SET: 842 file->f_pos = offset; 843 break; 844 case SEEK_CUR: 845 file->f_pos += offset; 846 break; 847 default: 848 return -EINVAL; 849 } 850 return file->f_pos; 851 } 852 853 static void init_pages_in_zone(struct zone *zone) 854 { 855 unsigned long pfn = zone->zone_start_pfn; 856 unsigned long end_pfn = zone_end_pfn(zone); 857 unsigned long count = 0; 858 859 /* 860 * Walk the zone in pageblock_nr_pages steps. If a page block spans 861 * a zone boundary, it will be double counted between zones. This does 862 * not matter as the mixed block count will still be correct 863 */ 864 for (; pfn < end_pfn; ) { 865 unsigned long block_end_pfn; 866 867 if (!pfn_valid(pfn)) { 868 pfn = ALIGN(pfn + 1, MAX_ORDER_NR_PAGES); 869 continue; 870 } 871 872 block_end_pfn = pageblock_end_pfn(pfn); 873 block_end_pfn = min(block_end_pfn, end_pfn); 874 875 for (; pfn < block_end_pfn; pfn++) { 876 struct page *page = pfn_to_page(pfn); 877 struct page_ext *page_ext; 878 879 if (page_zone(page) != zone) 880 continue; 881 882 if (skip_buddy_pages(&pfn, page)) 883 continue; 884 885 if (PageReserved(page)) 886 continue; 887 888 page_ext = page_ext_get(page); 889 if (unlikely(!page_ext)) 890 continue; 891 892 /* Maybe overlapping zone */ 893 if (test_bit(PAGE_EXT_OWNER, &page_ext->flags)) 894 goto ext_put_continue; 895 896 /* Found early allocated page */ 897 __update_page_owner_handle(page, early_handle, 0, 0, 898 MR_NEVER, local_clock(), current->pid, 899 current->tgid, current->comm); 900 count++; 901 ext_put_continue: 902 page_ext_put(page_ext); 903 } 904 cond_resched(); 905 } 906 907 pr_info("Node %d, zone %8s: page owner found early allocated %lu pages\n", 908 zone->zone_pgdat->node_id, zone->name, count); 909 } 910 911 static void init_early_allocated_pages(void) 912 { 913 struct zone *zone; 914 915 for_each_populated_zone(zone) 916 init_pages_in_zone(zone); 917 } 918 919 static int page_owner_open(struct inode *inode, struct file *file) 920 { 921 struct page_owner_filter_state *state; 922 923 state = kzalloc_obj(*state); 924 if (!state) 925 return -ENOMEM; 926 927 state->print_mode = PAGE_OWNER_PRINT_STACK; 928 nodes_clear(state->nid_filter); 929 state->nid_filter_enabled = false; 930 file->private_data = state; 931 return 0; 932 } 933 934 static int page_owner_release(struct inode *inode, struct file *file) 935 { 936 kfree(file->private_data); 937 return 0; 938 } 939 940 static ssize_t page_owner_write(struct file *file, 941 const char __user *buf, 942 size_t count, loff_t *ppos) 943 { 944 char *kbuf; 945 char *orig; 946 char *token; 947 int ret; 948 struct page_owner_filter_state *state = file->private_data; 949 enum page_owner_print_mode new_print_mode; 950 nodemask_t new_nid_filter; 951 bool new_nid_filter_enabled; 952 953 /* 954 * Maximum input length for filter commands: 955 * - 32: print_mode command max length is 17 ("mode=stack_handle") 956 * with sufficient buffer 957 * - 6 * MAX_NUMNODES: worst case for nid list 958 * Worst case per node: ",NNNNN" (comma + 5-digit node number) = 6 bytes 959 */ 960 if (count > 32 + 6 * MAX_NUMNODES) 961 return -EINVAL; 962 963 kbuf = memdup_user_nul(buf, count); 964 if (IS_ERR(kbuf)) 965 return PTR_ERR(kbuf); 966 967 orig = kbuf; 968 969 new_print_mode = state->print_mode; 970 new_nid_filter = state->nid_filter; 971 new_nid_filter_enabled = state->nid_filter_enabled; 972 973 while ((token = strsep(&kbuf, " \t\n")) != NULL) { 974 if (*token == '\0') 975 continue; 976 977 if (!strncmp(token, "mode=", 5)) { 978 ret = sysfs_match_string(page_owner_print_mode_strings, 979 token + 5); 980 if (ret < 0) 981 goto out_free; 982 new_print_mode = ret; 983 } else if (!strncmp(token, "nid=", 4)) { 984 ret = nodelist_parse(token + 4, new_nid_filter); 985 if (ret < 0) 986 goto out_free; 987 988 if (nodes_empty(new_nid_filter)) { 989 ret = -EINVAL; 990 goto out_free; 991 } 992 993 /* 994 * We want to filter memory allocations by numa nodes, so make sure 995 * that the specified nodes have memory. 996 */ 997 if (!nodes_subset(new_nid_filter, node_states[N_MEMORY])) { 998 ret = -EINVAL; 999 goto out_free; 1000 } 1001 1002 new_nid_filter_enabled = true; 1003 } else { 1004 ret = -EINVAL; 1005 goto out_free; 1006 } 1007 } 1008 1009 /* Commit all filter changes */ 1010 state->print_mode = new_print_mode; 1011 state->nid_filter = new_nid_filter; 1012 state->nid_filter_enabled = new_nid_filter_enabled; 1013 1014 ret = count; 1015 1016 out_free: 1017 kfree(orig); 1018 return ret; 1019 } 1020 1021 static const struct file_operations page_owner_fops = { 1022 .owner = THIS_MODULE, 1023 .open = page_owner_open, 1024 .release = page_owner_release, 1025 .write = page_owner_write, 1026 .read = read_page_owner, 1027 .llseek = lseek_page_owner, 1028 }; 1029 1030 static void *stack_start(struct seq_file *m, loff_t *ppos) 1031 { 1032 struct stack *stack; 1033 struct stack_print_ctx *ctx = m->private; 1034 1035 if (*ppos == -1UL) 1036 return NULL; 1037 1038 if (!*ppos) { 1039 /* 1040 * This pairs with smp_store_release() from function 1041 * add_stack_record_to_list(), so we get a consistent 1042 * value of stack_list. 1043 */ 1044 stack = smp_load_acquire(&stack_list); 1045 ctx->stack = stack; 1046 } else { 1047 stack = ctx->stack; 1048 } 1049 1050 return stack; 1051 } 1052 1053 static void *stack_next(struct seq_file *m, void *v, loff_t *ppos) 1054 { 1055 struct stack *stack = v; 1056 struct stack_print_ctx *ctx = m->private; 1057 1058 stack = stack->next; 1059 *ppos = stack ? *ppos + 1 : -1UL; 1060 ctx->stack = stack; 1061 1062 return stack; 1063 } 1064 1065 static unsigned long pages_threshold; 1066 1067 static int stack_print(struct seq_file *m, void *v) 1068 { 1069 int i, nr_base_pages; 1070 struct stack *stack = v; 1071 unsigned long *entries; 1072 unsigned long nr_entries; 1073 struct stack_record *stack_record = stack->stack_record; 1074 struct stack_print_ctx *ctx = m->private; 1075 1076 if (!stack->stack_record) 1077 return 0; 1078 1079 nr_base_pages = refcount_read(&stack_record->count) - 1; 1080 1081 if (ctx->flags & STACK_PRINT_FLAG_PAGES && 1082 (nr_base_pages < 1 || nr_base_pages < pages_threshold)) 1083 return 0; 1084 1085 if (ctx->flags & STACK_PRINT_FLAG_STACK) { 1086 nr_entries = stack_record->size; 1087 entries = stack_record->entries; 1088 for (i = 0; i < nr_entries; i++) 1089 seq_printf(m, " %pS\n", (void *)entries[i]); 1090 } 1091 if (ctx->flags & STACK_PRINT_FLAG_HANDLE) 1092 seq_printf(m, "handle: %d\n", stack_record->handle.handle); 1093 if (ctx->flags & STACK_PRINT_FLAG_PAGES) 1094 seq_printf(m, "nr_base_pages: %d\n", nr_base_pages); 1095 seq_putc(m, '\n'); 1096 1097 return 0; 1098 } 1099 1100 static void stack_stop(struct seq_file *m, void *v) 1101 { 1102 } 1103 1104 static const struct seq_operations stack_op = { 1105 .start = stack_start, 1106 .next = stack_next, 1107 .stop = stack_stop, 1108 .show = stack_print 1109 }; 1110 1111 static int stack_open(struct inode *inode, struct file *file) 1112 { 1113 int ret = seq_open_private(file, &stack_op, 1114 sizeof(struct stack_print_ctx)); 1115 1116 if (!ret) { 1117 struct seq_file *m = file->private_data; 1118 struct stack_print_ctx *ctx = m->private; 1119 1120 ctx->flags = (uintptr_t) inode->i_private; 1121 } 1122 1123 return ret; 1124 } 1125 1126 static const struct file_operations stack_fops = { 1127 .open = stack_open, 1128 .read = seq_read, 1129 .llseek = seq_lseek, 1130 .release = seq_release_private, 1131 }; 1132 1133 static int threshold_get(void *data, u64 *val) 1134 { 1135 *val = READ_ONCE(pages_threshold); 1136 return 0; 1137 } 1138 1139 static int threshold_set(void *data, u64 val) 1140 { 1141 WRITE_ONCE(pages_threshold, val); 1142 return 0; 1143 } 1144 1145 DEFINE_SIMPLE_ATTRIBUTE(threshold_fops, &threshold_get, &threshold_set, "%llu\n"); 1146 1147 static int __init pageowner_init(void) 1148 { 1149 struct dentry *dir; 1150 1151 if (!static_branch_unlikely(&page_owner_inited)) { 1152 pr_info("page_owner is disabled\n"); 1153 return 0; 1154 } 1155 1156 debugfs_create_file("page_owner", 0600, NULL, NULL, &page_owner_fops); 1157 dir = debugfs_create_dir("page_owner_stacks", NULL); 1158 debugfs_create_file("show_stacks", 0400, dir, 1159 (void *)(STACK_PRINT_FLAG_STACK | 1160 STACK_PRINT_FLAG_PAGES), 1161 &stack_fops); 1162 debugfs_create_file("show_handles", 0400, dir, 1163 (void *)(STACK_PRINT_FLAG_HANDLE | 1164 STACK_PRINT_FLAG_PAGES), 1165 &stack_fops); 1166 debugfs_create_file("show_stacks_handles", 0400, dir, 1167 (void *)(STACK_PRINT_FLAG_STACK | 1168 STACK_PRINT_FLAG_HANDLE), 1169 &stack_fops); 1170 debugfs_create_file("count_threshold", 0600, dir, NULL, 1171 &threshold_fops); 1172 return 0; 1173 } 1174 late_initcall(pageowner_init) 1175