1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 248c96a36SJoonsoo Kim #include <linux/debugfs.h> 348c96a36SJoonsoo Kim #include <linux/mm.h> 448c96a36SJoonsoo Kim #include <linux/slab.h> 548c96a36SJoonsoo Kim #include <linux/uaccess.h> 657c8a661SMike Rapoport #include <linux/memblock.h> 748c96a36SJoonsoo Kim #include <linux/stacktrace.h> 848c96a36SJoonsoo Kim #include <linux/page_owner.h> 97dd80b8aSVlastimil Babka #include <linux/jump_label.h> 107cd12b4aSVlastimil Babka #include <linux/migrate.h> 11f2ca0b55SJoonsoo Kim #include <linux/stackdepot.h> 12e2f612e6SJoonsoo Kim #include <linux/seq_file.h> 13fcf89358SWaiman Long #include <linux/memcontrol.h> 149cc7e96aSLiam Mark #include <linux/sched/clock.h> 15f2ca0b55SJoonsoo Kim 1648c96a36SJoonsoo Kim #include "internal.h" 1748c96a36SJoonsoo Kim 18f2ca0b55SJoonsoo Kim /* 19f2ca0b55SJoonsoo Kim * TODO: teach PAGE_OWNER_STACK_DEPTH (__dump_page_owner and save_stack) 20f2ca0b55SJoonsoo Kim * to use off stack temporal storage 21f2ca0b55SJoonsoo Kim */ 22f2ca0b55SJoonsoo Kim #define PAGE_OWNER_STACK_DEPTH (16) 23f2ca0b55SJoonsoo Kim 249300d8dfSJoonsoo Kim struct page_owner { 256b4c54e3SAyush Mittal unsigned short order; 266b4c54e3SAyush Mittal short last_migrate_reason; 279300d8dfSJoonsoo Kim gfp_t gfp_mask; 289300d8dfSJoonsoo Kim depot_stack_handle_t handle; 298974558fSVlastimil Babka depot_stack_handle_t free_handle; 309cc7e96aSLiam Mark u64 ts_nsec; 31866b4852SGeorgi Djakov u64 free_ts_nsec; 32865ed6a3SWaiman Long char comm[TASK_COMM_LEN]; 339cc7e96aSLiam Mark pid_t pid; 34bf215eabSYixuan Cao pid_t tgid; 359300d8dfSJoonsoo Kim }; 369300d8dfSJoonsoo Kim 373645b5ecSFanjun Kong static bool page_owner_enabled __initdata; 387dd80b8aSVlastimil Babka DEFINE_STATIC_KEY_FALSE(page_owner_inited); 3948c96a36SJoonsoo Kim 40f2ca0b55SJoonsoo Kim static depot_stack_handle_t dummy_handle; 41f2ca0b55SJoonsoo Kim static depot_stack_handle_t failure_handle; 42dab4ead1SVlastimil Babka static depot_stack_handle_t early_handle; 43f2ca0b55SJoonsoo Kim 4461cf5febSJoonsoo Kim static void init_early_allocated_pages(void); 4561cf5febSJoonsoo Kim 461173194eSDou Liyang static int __init early_page_owner_param(char *buf) 4748c96a36SJoonsoo Kim { 48a5f1783bSVlastimil Babka int ret = kstrtobool(buf, &page_owner_enabled); 49a5f1783bSVlastimil Babka 50a5f1783bSVlastimil Babka if (page_owner_enabled) 51a5f1783bSVlastimil Babka stack_depot_want_early_init(); 52a5f1783bSVlastimil Babka 53a5f1783bSVlastimil Babka return ret; 5448c96a36SJoonsoo Kim } 5548c96a36SJoonsoo Kim early_param("page_owner", early_page_owner_param); 5648c96a36SJoonsoo Kim 57cab0a7c1STing Liu static __init bool need_page_owner(void) 5848c96a36SJoonsoo Kim { 590fe9a448SVlastimil Babka return page_owner_enabled; 6048c96a36SJoonsoo Kim } 6148c96a36SJoonsoo Kim 62dab4ead1SVlastimil Babka static __always_inline depot_stack_handle_t create_dummy_stack(void) 63f2ca0b55SJoonsoo Kim { 64f2ca0b55SJoonsoo Kim unsigned long entries[4]; 65af52bf6bSThomas Gleixner unsigned int nr_entries; 66f2ca0b55SJoonsoo Kim 67af52bf6bSThomas Gleixner nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 0); 68af52bf6bSThomas Gleixner return stack_depot_save(entries, nr_entries, GFP_KERNEL); 69dab4ead1SVlastimil Babka } 70dab4ead1SVlastimil Babka 71dab4ead1SVlastimil Babka static noinline void register_dummy_stack(void) 72dab4ead1SVlastimil Babka { 73dab4ead1SVlastimil Babka dummy_handle = create_dummy_stack(); 74f2ca0b55SJoonsoo Kim } 75f2ca0b55SJoonsoo Kim 76f2ca0b55SJoonsoo Kim static noinline void register_failure_stack(void) 77f2ca0b55SJoonsoo Kim { 78dab4ead1SVlastimil Babka failure_handle = create_dummy_stack(); 79dab4ead1SVlastimil Babka } 80f2ca0b55SJoonsoo Kim 81dab4ead1SVlastimil Babka static noinline void register_early_stack(void) 82dab4ead1SVlastimil Babka { 83dab4ead1SVlastimil Babka early_handle = create_dummy_stack(); 84f2ca0b55SJoonsoo Kim } 85f2ca0b55SJoonsoo Kim 86cab0a7c1STing Liu static __init void init_page_owner(void) 8748c96a36SJoonsoo Kim { 880fe9a448SVlastimil Babka if (!page_owner_enabled) 8948c96a36SJoonsoo Kim return; 9048c96a36SJoonsoo Kim 91f2ca0b55SJoonsoo Kim register_dummy_stack(); 92f2ca0b55SJoonsoo Kim register_failure_stack(); 93dab4ead1SVlastimil Babka register_early_stack(); 947dd80b8aSVlastimil Babka static_branch_enable(&page_owner_inited); 9561cf5febSJoonsoo Kim init_early_allocated_pages(); 9648c96a36SJoonsoo Kim } 9748c96a36SJoonsoo Kim 9848c96a36SJoonsoo Kim struct page_ext_operations page_owner_ops = { 999300d8dfSJoonsoo Kim .size = sizeof(struct page_owner), 10048c96a36SJoonsoo Kim .need = need_page_owner, 10148c96a36SJoonsoo Kim .init = init_page_owner, 10248c96a36SJoonsoo Kim }; 10348c96a36SJoonsoo Kim 1049300d8dfSJoonsoo Kim static inline struct page_owner *get_page_owner(struct page_ext *page_ext) 1059300d8dfSJoonsoo Kim { 1069300d8dfSJoonsoo Kim return (void *)page_ext + page_owner_ops.offset; 1079300d8dfSJoonsoo Kim } 1089300d8dfSJoonsoo Kim 109f2ca0b55SJoonsoo Kim static noinline depot_stack_handle_t save_stack(gfp_t flags) 110f2ca0b55SJoonsoo Kim { 111f2ca0b55SJoonsoo Kim unsigned long entries[PAGE_OWNER_STACK_DEPTH]; 112f2ca0b55SJoonsoo Kim depot_stack_handle_t handle; 113af52bf6bSThomas Gleixner unsigned int nr_entries; 114f2ca0b55SJoonsoo Kim 115f2ca0b55SJoonsoo Kim /* 1168e9b16c4SSergei Trofimovich * Avoid recursion. 1178e9b16c4SSergei Trofimovich * 1188e9b16c4SSergei Trofimovich * Sometimes page metadata allocation tracking requires more 1198e9b16c4SSergei Trofimovich * memory to be allocated: 1208e9b16c4SSergei Trofimovich * - when new stack trace is saved to stack depot 1218e9b16c4SSergei Trofimovich * - when backtrace itself is calculated (ia64) 122f2ca0b55SJoonsoo Kim */ 1238e9b16c4SSergei Trofimovich if (current->in_page_owner) 124f2ca0b55SJoonsoo Kim return dummy_handle; 1258e9b16c4SSergei Trofimovich current->in_page_owner = 1; 126f2ca0b55SJoonsoo Kim 1278e9b16c4SSergei Trofimovich nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 2); 128af52bf6bSThomas Gleixner handle = stack_depot_save(entries, nr_entries, flags); 129f2ca0b55SJoonsoo Kim if (!handle) 130f2ca0b55SJoonsoo Kim handle = failure_handle; 131f2ca0b55SJoonsoo Kim 1328e9b16c4SSergei Trofimovich current->in_page_owner = 0; 133f2ca0b55SJoonsoo Kim return handle; 134f2ca0b55SJoonsoo Kim } 135f2ca0b55SJoonsoo Kim 1360093de69SYixuan Cao void __reset_page_owner(struct page *page, unsigned short order) 1378974558fSVlastimil Babka { 1388974558fSVlastimil Babka int i; 1398974558fSVlastimil Babka struct page_ext *page_ext; 140fab765c2SSergei Trofimovich depot_stack_handle_t handle; 1418974558fSVlastimil Babka struct page_owner *page_owner; 142866b4852SGeorgi Djakov u64 free_ts_nsec = local_clock(); 1438974558fSVlastimil Babka 144b1d5488aSCharan Teja Kalla page_ext = page_ext_get(page); 1458974558fSVlastimil Babka if (unlikely(!page_ext)) 1465556cfe8SVlastimil Babka return; 147fab765c2SSergei Trofimovich 148fab765c2SSergei Trofimovich handle = save_stack(GFP_NOWAIT | __GFP_NOWARN); 1495556cfe8SVlastimil Babka for (i = 0; i < (1 << order); i++) { 150fdf3bf80SVlastimil Babka __clear_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags); 1518974558fSVlastimil Babka page_owner = get_page_owner(page_ext); 1528974558fSVlastimil Babka page_owner->free_handle = handle; 153866b4852SGeorgi Djakov page_owner->free_ts_nsec = free_ts_nsec; 1545556cfe8SVlastimil Babka page_ext = page_ext_next(page_ext); 1558974558fSVlastimil Babka } 156b1d5488aSCharan Teja Kalla page_ext_put(page_ext); 1578974558fSVlastimil Babka } 1588974558fSVlastimil Babka 15964ea78d2Szhongjiang-ali static inline void __set_page_owner_handle(struct page_ext *page_ext, 16064ea78d2Szhongjiang-ali depot_stack_handle_t handle, 1610093de69SYixuan Cao unsigned short order, gfp_t gfp_mask) 162f2ca0b55SJoonsoo Kim { 1639300d8dfSJoonsoo Kim struct page_owner *page_owner; 1647e2f2a0cSVlastimil Babka int i; 16548c96a36SJoonsoo Kim 1667e2f2a0cSVlastimil Babka for (i = 0; i < (1 << order); i++) { 1679300d8dfSJoonsoo Kim page_owner = get_page_owner(page_ext); 168dab4ead1SVlastimil Babka page_owner->handle = handle; 1699300d8dfSJoonsoo Kim page_owner->order = order; 1709300d8dfSJoonsoo Kim page_owner->gfp_mask = gfp_mask; 1719300d8dfSJoonsoo Kim page_owner->last_migrate_reason = -1; 1729cc7e96aSLiam Mark page_owner->pid = current->pid; 173bf215eabSYixuan Cao page_owner->tgid = current->tgid; 1749cc7e96aSLiam Mark page_owner->ts_nsec = local_clock(); 175cd8c1fd8SEric Dumazet strscpy(page_owner->comm, current->comm, 176865ed6a3SWaiman Long sizeof(page_owner->comm)); 17748c96a36SJoonsoo Kim __set_bit(PAGE_EXT_OWNER, &page_ext->flags); 178fdf3bf80SVlastimil Babka __set_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags); 1797e2f2a0cSVlastimil Babka 1805556cfe8SVlastimil Babka page_ext = page_ext_next(page_ext); 1817e2f2a0cSVlastimil Babka } 18248c96a36SJoonsoo Kim } 18348c96a36SJoonsoo Kim 1840093de69SYixuan Cao noinline void __set_page_owner(struct page *page, unsigned short order, 185dab4ead1SVlastimil Babka gfp_t gfp_mask) 186dab4ead1SVlastimil Babka { 187b1d5488aSCharan Teja Kalla struct page_ext *page_ext; 188dab4ead1SVlastimil Babka depot_stack_handle_t handle; 189dab4ead1SVlastimil Babka 190b1d5488aSCharan Teja Kalla handle = save_stack(gfp_mask); 191b1d5488aSCharan Teja Kalla 192b1d5488aSCharan Teja Kalla page_ext = page_ext_get(page); 193dab4ead1SVlastimil Babka if (unlikely(!page_ext)) 194dab4ead1SVlastimil Babka return; 19564ea78d2Szhongjiang-ali __set_page_owner_handle(page_ext, handle, order, gfp_mask); 196b1d5488aSCharan Teja Kalla page_ext_put(page_ext); 197dab4ead1SVlastimil Babka } 198dab4ead1SVlastimil Babka 1997cd12b4aSVlastimil Babka void __set_page_owner_migrate_reason(struct page *page, int reason) 2007cd12b4aSVlastimil Babka { 201b1d5488aSCharan Teja Kalla struct page_ext *page_ext = page_ext_get(page); 2029300d8dfSJoonsoo Kim struct page_owner *page_owner; 2039300d8dfSJoonsoo Kim 204f86e4271SYang Shi if (unlikely(!page_ext)) 205f86e4271SYang Shi return; 2067cd12b4aSVlastimil Babka 2079300d8dfSJoonsoo Kim page_owner = get_page_owner(page_ext); 2089300d8dfSJoonsoo Kim page_owner->last_migrate_reason = reason; 209b1d5488aSCharan Teja Kalla page_ext_put(page_ext); 2107cd12b4aSVlastimil Babka } 2117cd12b4aSVlastimil Babka 2128fb156c9SMatthew Wilcox (Oracle) void __split_page_owner(struct page *page, unsigned int nr) 213e2cfc911SJoonsoo Kim { 214a9627bc5SJoonsoo Kim int i; 215b1d5488aSCharan Teja Kalla struct page_ext *page_ext = page_ext_get(page); 2169300d8dfSJoonsoo Kim struct page_owner *page_owner; 217e2cfc911SJoonsoo Kim 218a9627bc5SJoonsoo Kim if (unlikely(!page_ext)) 219a9627bc5SJoonsoo Kim return; 220a9627bc5SJoonsoo Kim 2218fb156c9SMatthew Wilcox (Oracle) for (i = 0; i < nr; i++) { 2229300d8dfSJoonsoo Kim page_owner = get_page_owner(page_ext); 2239300d8dfSJoonsoo Kim page_owner->order = 0; 2245556cfe8SVlastimil Babka page_ext = page_ext_next(page_ext); 2257e2f2a0cSVlastimil Babka } 226b1d5488aSCharan Teja Kalla page_ext_put(page_ext); 227e2cfc911SJoonsoo Kim } 228e2cfc911SJoonsoo Kim 22919138349SMatthew Wilcox (Oracle) void __folio_copy_owner(struct folio *newfolio, struct folio *old) 230d435edcaSVlastimil Babka { 231b1d5488aSCharan Teja Kalla struct page_ext *old_ext; 232b1d5488aSCharan Teja Kalla struct page_ext *new_ext; 2339300d8dfSJoonsoo Kim struct page_owner *old_page_owner, *new_page_owner; 234d435edcaSVlastimil Babka 235b1d5488aSCharan Teja Kalla old_ext = page_ext_get(&old->page); 236b1d5488aSCharan Teja Kalla if (unlikely(!old_ext)) 237f86e4271SYang Shi return; 238f86e4271SYang Shi 239b1d5488aSCharan Teja Kalla new_ext = page_ext_get(&newfolio->page); 240b1d5488aSCharan Teja Kalla if (unlikely(!new_ext)) { 241b1d5488aSCharan Teja Kalla page_ext_put(old_ext); 242b1d5488aSCharan Teja Kalla return; 243b1d5488aSCharan Teja Kalla } 244b1d5488aSCharan Teja Kalla 2459300d8dfSJoonsoo Kim old_page_owner = get_page_owner(old_ext); 2469300d8dfSJoonsoo Kim new_page_owner = get_page_owner(new_ext); 2479300d8dfSJoonsoo Kim new_page_owner->order = old_page_owner->order; 2489300d8dfSJoonsoo Kim new_page_owner->gfp_mask = old_page_owner->gfp_mask; 2499300d8dfSJoonsoo Kim new_page_owner->last_migrate_reason = 2509300d8dfSJoonsoo Kim old_page_owner->last_migrate_reason; 2519300d8dfSJoonsoo Kim new_page_owner->handle = old_page_owner->handle; 2529cc7e96aSLiam Mark new_page_owner->pid = old_page_owner->pid; 253bf215eabSYixuan Cao new_page_owner->tgid = old_page_owner->tgid; 2549cc7e96aSLiam Mark new_page_owner->ts_nsec = old_page_owner->ts_nsec; 255866b4852SGeorgi Djakov new_page_owner->free_ts_nsec = old_page_owner->ts_nsec; 256865ed6a3SWaiman Long strcpy(new_page_owner->comm, old_page_owner->comm); 257d435edcaSVlastimil Babka 258d435edcaSVlastimil Babka /* 25919138349SMatthew Wilcox (Oracle) * We don't clear the bit on the old folio as it's going to be freed 260d435edcaSVlastimil Babka * after migration. Until then, the info can be useful in case of 261f0953a1bSIngo Molnar * a bug, and the overall stats will be off a bit only temporarily. 262d435edcaSVlastimil Babka * Also, migrate_misplaced_transhuge_page() can still fail the 26319138349SMatthew Wilcox (Oracle) * migration and then we want the old folio to retain the info. But 264d435edcaSVlastimil Babka * in that case we also don't need to explicitly clear the info from 265d435edcaSVlastimil Babka * the new page, which will be freed. 266d435edcaSVlastimil Babka */ 267d435edcaSVlastimil Babka __set_bit(PAGE_EXT_OWNER, &new_ext->flags); 268fdf3bf80SVlastimil Babka __set_bit(PAGE_EXT_OWNER_ALLOCATED, &new_ext->flags); 269b1d5488aSCharan Teja Kalla page_ext_put(new_ext); 270b1d5488aSCharan Teja Kalla page_ext_put(old_ext); 271d435edcaSVlastimil Babka } 272d435edcaSVlastimil Babka 273e2f612e6SJoonsoo Kim void pagetypeinfo_showmixedcount_print(struct seq_file *m, 274e2f612e6SJoonsoo Kim pg_data_t *pgdat, struct zone *zone) 275e2f612e6SJoonsoo Kim { 276e2f612e6SJoonsoo Kim struct page *page; 277e2f612e6SJoonsoo Kim struct page_ext *page_ext; 2789300d8dfSJoonsoo Kim struct page_owner *page_owner; 2791d2cae8eSMiaohe Lin unsigned long pfn, block_end_pfn; 2801d2cae8eSMiaohe Lin unsigned long end_pfn = zone_end_pfn(zone); 281e2f612e6SJoonsoo Kim unsigned long count[MIGRATE_TYPES] = { 0, }; 282e2f612e6SJoonsoo Kim int pageblock_mt, page_mt; 283e2f612e6SJoonsoo Kim int i; 284e2f612e6SJoonsoo Kim 285e2f612e6SJoonsoo Kim /* Scan block by block. First and last block may be incomplete */ 286e2f612e6SJoonsoo Kim pfn = zone->zone_start_pfn; 287e2f612e6SJoonsoo Kim 288e2f612e6SJoonsoo Kim /* 289e2f612e6SJoonsoo Kim * Walk the zone in pageblock_nr_pages steps. If a page block spans 290e2f612e6SJoonsoo Kim * a zone boundary, it will be double counted between zones. This does 291e2f612e6SJoonsoo Kim * not matter as the mixed block count will still be correct 292e2f612e6SJoonsoo Kim */ 293e2f612e6SJoonsoo Kim for (; pfn < end_pfn; ) { 294a26ee565SQian Cai page = pfn_to_online_page(pfn); 295a26ee565SQian Cai if (!page) { 296e2f612e6SJoonsoo Kim pfn = ALIGN(pfn + 1, MAX_ORDER_NR_PAGES); 297e2f612e6SJoonsoo Kim continue; 298e2f612e6SJoonsoo Kim } 299e2f612e6SJoonsoo Kim 300*4f9bc69aSKefeng Wang block_end_pfn = pageblock_end_pfn(pfn); 301e2f612e6SJoonsoo Kim block_end_pfn = min(block_end_pfn, end_pfn); 302e2f612e6SJoonsoo Kim 303e2f612e6SJoonsoo Kim pageblock_mt = get_pageblock_migratetype(page); 304e2f612e6SJoonsoo Kim 305e2f612e6SJoonsoo Kim for (; pfn < block_end_pfn; pfn++) { 306a26ee565SQian Cai /* The pageblock is online, no need to recheck. */ 307e2f612e6SJoonsoo Kim page = pfn_to_page(pfn); 308e2f612e6SJoonsoo Kim 309e2f612e6SJoonsoo Kim if (page_zone(page) != zone) 310e2f612e6SJoonsoo Kim continue; 311e2f612e6SJoonsoo Kim 312e2f612e6SJoonsoo Kim if (PageBuddy(page)) { 313727c080fSVinayak Menon unsigned long freepage_order; 314727c080fSVinayak Menon 315ab130f91SMatthew Wilcox (Oracle) freepage_order = buddy_order_unsafe(page); 316727c080fSVinayak Menon if (freepage_order < MAX_ORDER) 317727c080fSVinayak Menon pfn += (1UL << freepage_order) - 1; 318e2f612e6SJoonsoo Kim continue; 319e2f612e6SJoonsoo Kim } 320e2f612e6SJoonsoo Kim 321e2f612e6SJoonsoo Kim if (PageReserved(page)) 322e2f612e6SJoonsoo Kim continue; 323e2f612e6SJoonsoo Kim 324b1d5488aSCharan Teja Kalla page_ext = page_ext_get(page); 325e2f612e6SJoonsoo Kim if (unlikely(!page_ext)) 326e2f612e6SJoonsoo Kim continue; 327e2f612e6SJoonsoo Kim 328fdf3bf80SVlastimil Babka if (!test_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags)) 329b1d5488aSCharan Teja Kalla goto ext_put_continue; 330e2f612e6SJoonsoo Kim 3319300d8dfSJoonsoo Kim page_owner = get_page_owner(page_ext); 33201c0bfe0SWei Yang page_mt = gfp_migratetype(page_owner->gfp_mask); 333e2f612e6SJoonsoo Kim if (pageblock_mt != page_mt) { 334e2f612e6SJoonsoo Kim if (is_migrate_cma(pageblock_mt)) 335e2f612e6SJoonsoo Kim count[MIGRATE_MOVABLE]++; 336e2f612e6SJoonsoo Kim else 337e2f612e6SJoonsoo Kim count[pageblock_mt]++; 338e2f612e6SJoonsoo Kim 339e2f612e6SJoonsoo Kim pfn = block_end_pfn; 340b1d5488aSCharan Teja Kalla page_ext_put(page_ext); 341e2f612e6SJoonsoo Kim break; 342e2f612e6SJoonsoo Kim } 3439300d8dfSJoonsoo Kim pfn += (1UL << page_owner->order) - 1; 344b1d5488aSCharan Teja Kalla ext_put_continue: 345b1d5488aSCharan Teja Kalla page_ext_put(page_ext); 346e2f612e6SJoonsoo Kim } 347e2f612e6SJoonsoo Kim } 348e2f612e6SJoonsoo Kim 349e2f612e6SJoonsoo Kim /* Print counts */ 350e2f612e6SJoonsoo Kim seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name); 351e2f612e6SJoonsoo Kim for (i = 0; i < MIGRATE_TYPES; i++) 352e2f612e6SJoonsoo Kim seq_printf(m, "%12lu ", count[i]); 353e2f612e6SJoonsoo Kim seq_putc(m, '\n'); 354e2f612e6SJoonsoo Kim } 355e2f612e6SJoonsoo Kim 356fcf89358SWaiman Long /* 357fcf89358SWaiman Long * Looking for memcg information and print it out 358fcf89358SWaiman Long */ 359fcf89358SWaiman Long static inline int print_page_owner_memcg(char *kbuf, size_t count, int ret, 360fcf89358SWaiman Long struct page *page) 361fcf89358SWaiman Long { 362fcf89358SWaiman Long #ifdef CONFIG_MEMCG 363fcf89358SWaiman Long unsigned long memcg_data; 364fcf89358SWaiman Long struct mem_cgroup *memcg; 365fcf89358SWaiman Long bool online; 366fcf89358SWaiman Long char name[80]; 367fcf89358SWaiman Long 368fcf89358SWaiman Long rcu_read_lock(); 369fcf89358SWaiman Long memcg_data = READ_ONCE(page->memcg_data); 370fcf89358SWaiman Long if (!memcg_data) 371fcf89358SWaiman Long goto out_unlock; 372fcf89358SWaiman Long 373fcf89358SWaiman Long if (memcg_data & MEMCG_DATA_OBJCGS) 374fcf89358SWaiman Long ret += scnprintf(kbuf + ret, count - ret, 375fcf89358SWaiman Long "Slab cache page\n"); 376fcf89358SWaiman Long 377fcf89358SWaiman Long memcg = page_memcg_check(page); 378fcf89358SWaiman Long if (!memcg) 379fcf89358SWaiman Long goto out_unlock; 380fcf89358SWaiman Long 381fcf89358SWaiman Long online = (memcg->css.flags & CSS_ONLINE); 382fcf89358SWaiman Long cgroup_name(memcg->css.cgroup, name, sizeof(name)); 383fcf89358SWaiman Long ret += scnprintf(kbuf + ret, count - ret, 384fcf89358SWaiman Long "Charged %sto %smemcg %s\n", 385fcf89358SWaiman Long PageMemcgKmem(page) ? "(via objcg) " : "", 386fcf89358SWaiman Long online ? "" : "offline ", 387fcf89358SWaiman Long name); 388fcf89358SWaiman Long out_unlock: 389fcf89358SWaiman Long rcu_read_unlock(); 390fcf89358SWaiman Long #endif /* CONFIG_MEMCG */ 391fcf89358SWaiman Long 392fcf89358SWaiman Long return ret; 393fcf89358SWaiman Long } 394fcf89358SWaiman Long 39548c96a36SJoonsoo Kim static ssize_t 39648c96a36SJoonsoo Kim print_page_owner(char __user *buf, size_t count, unsigned long pfn, 3979300d8dfSJoonsoo Kim struct page *page, struct page_owner *page_owner, 398f2ca0b55SJoonsoo Kim depot_stack_handle_t handle) 39948c96a36SJoonsoo Kim { 400af52bf6bSThomas Gleixner int ret, pageblock_mt, page_mt; 40148c96a36SJoonsoo Kim char *kbuf; 40248c96a36SJoonsoo Kim 403c8f61cfcSMiles Chen count = min_t(size_t, count, PAGE_SIZE); 40448c96a36SJoonsoo Kim kbuf = kmalloc(count, GFP_KERNEL); 40548c96a36SJoonsoo Kim if (!kbuf) 40648c96a36SJoonsoo Kim return -ENOMEM; 40748c96a36SJoonsoo Kim 4083ebc4397SWaiman Long ret = scnprintf(kbuf, count, 409bf215eabSYixuan Cao "Page allocated via order %u, mask %#x(%pGg), pid %d, tgid %d (%s), ts %llu ns, free_ts %llu ns\n", 4109300d8dfSJoonsoo Kim page_owner->order, page_owner->gfp_mask, 4119cc7e96aSLiam Mark &page_owner->gfp_mask, page_owner->pid, 412bf215eabSYixuan Cao page_owner->tgid, page_owner->comm, 413bf215eabSYixuan Cao page_owner->ts_nsec, page_owner->free_ts_nsec); 41448c96a36SJoonsoo Kim 41548c96a36SJoonsoo Kim /* Print information relevant to grouping pages by mobility */ 4160b423ca2SMel Gorman pageblock_mt = get_pageblock_migratetype(page); 41701c0bfe0SWei Yang page_mt = gfp_migratetype(page_owner->gfp_mask); 4183ebc4397SWaiman Long ret += scnprintf(kbuf + ret, count - ret, 41923efd080SMatthew Wilcox (Oracle) "PFN %lu type %s Block %lu type %s Flags %pGp\n", 42048c96a36SJoonsoo Kim pfn, 42160f30350SVlastimil Babka migratetype_names[page_mt], 42248c96a36SJoonsoo Kim pfn >> pageblock_order, 42360f30350SVlastimil Babka migratetype_names[pageblock_mt], 42423efd080SMatthew Wilcox (Oracle) &page->flags); 42548c96a36SJoonsoo Kim 4260f68d45eSImran Khan ret += stack_depot_snprint(handle, kbuf + ret, count - ret, 0); 42748c96a36SJoonsoo Kim if (ret >= count) 42848c96a36SJoonsoo Kim goto err; 42948c96a36SJoonsoo Kim 4309300d8dfSJoonsoo Kim if (page_owner->last_migrate_reason != -1) { 4313ebc4397SWaiman Long ret += scnprintf(kbuf + ret, count - ret, 4327cd12b4aSVlastimil Babka "Page has been migrated, last migrate reason: %s\n", 4339300d8dfSJoonsoo Kim migrate_reason_names[page_owner->last_migrate_reason]); 4347cd12b4aSVlastimil Babka } 4357cd12b4aSVlastimil Babka 436fcf89358SWaiman Long ret = print_page_owner_memcg(kbuf, count, ret, page); 437fcf89358SWaiman Long 43848c96a36SJoonsoo Kim ret += snprintf(kbuf + ret, count - ret, "\n"); 43948c96a36SJoonsoo Kim if (ret >= count) 44048c96a36SJoonsoo Kim goto err; 44148c96a36SJoonsoo Kim 44248c96a36SJoonsoo Kim if (copy_to_user(buf, kbuf, ret)) 44348c96a36SJoonsoo Kim ret = -EFAULT; 44448c96a36SJoonsoo Kim 44548c96a36SJoonsoo Kim kfree(kbuf); 44648c96a36SJoonsoo Kim return ret; 44748c96a36SJoonsoo Kim 44848c96a36SJoonsoo Kim err: 44948c96a36SJoonsoo Kim kfree(kbuf); 45048c96a36SJoonsoo Kim return -ENOMEM; 45148c96a36SJoonsoo Kim } 45248c96a36SJoonsoo Kim 4538bf6f451SMatthew Wilcox (Oracle) void __dump_page_owner(const struct page *page) 4544e462112SVlastimil Babka { 455b1d5488aSCharan Teja Kalla struct page_ext *page_ext = page_ext_get((void *)page); 4569300d8dfSJoonsoo Kim struct page_owner *page_owner; 457f2ca0b55SJoonsoo Kim depot_stack_handle_t handle; 4588285027fSSudip Mukherjee gfp_t gfp_mask; 4598285027fSSudip Mukherjee int mt; 4604e462112SVlastimil Babka 461f86e4271SYang Shi if (unlikely(!page_ext)) { 462f86e4271SYang Shi pr_alert("There is not page extension available.\n"); 463f86e4271SYang Shi return; 464f86e4271SYang Shi } 4659300d8dfSJoonsoo Kim 4669300d8dfSJoonsoo Kim page_owner = get_page_owner(page_ext); 4679300d8dfSJoonsoo Kim gfp_mask = page_owner->gfp_mask; 46801c0bfe0SWei Yang mt = gfp_migratetype(gfp_mask); 469f86e4271SYang Shi 4704e462112SVlastimil Babka if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags)) { 47137389167SVlastimil Babka pr_alert("page_owner info is not present (never set?)\n"); 472b1d5488aSCharan Teja Kalla page_ext_put(page_ext); 4734e462112SVlastimil Babka return; 4744e462112SVlastimil Babka } 4754e462112SVlastimil Babka 476fdf3bf80SVlastimil Babka if (test_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags)) 47737389167SVlastimil Babka pr_alert("page_owner tracks the page as allocated\n"); 47837389167SVlastimil Babka else 47937389167SVlastimil Babka pr_alert("page_owner tracks the page as freed\n"); 48037389167SVlastimil Babka 481bf215eabSYixuan Cao pr_alert("page last allocated via order %u, migratetype %s, gfp_mask %#x(%pGg), pid %d, tgid %d (%s), ts %llu, free_ts %llu\n", 4829cc7e96aSLiam Mark page_owner->order, migratetype_names[mt], gfp_mask, &gfp_mask, 483bf215eabSYixuan Cao page_owner->pid, page_owner->tgid, page_owner->comm, 484bf215eabSYixuan Cao page_owner->ts_nsec, page_owner->free_ts_nsec); 48537389167SVlastimil Babka 4869300d8dfSJoonsoo Kim handle = READ_ONCE(page_owner->handle); 487505be481SImran Khan if (!handle) 48837389167SVlastimil Babka pr_alert("page_owner allocation stack trace missing\n"); 489505be481SImran Khan else 490505be481SImran Khan stack_depot_print(handle); 4914e462112SVlastimil Babka 4928974558fSVlastimil Babka handle = READ_ONCE(page_owner->free_handle); 4938974558fSVlastimil Babka if (!handle) { 4948974558fSVlastimil Babka pr_alert("page_owner free stack trace missing\n"); 4958974558fSVlastimil Babka } else { 4968974558fSVlastimil Babka pr_alert("page last free stack trace:\n"); 497505be481SImran Khan stack_depot_print(handle); 4988974558fSVlastimil Babka } 4998974558fSVlastimil Babka 5009300d8dfSJoonsoo Kim if (page_owner->last_migrate_reason != -1) 5014e462112SVlastimil Babka pr_alert("page has been migrated, last migrate reason: %s\n", 5029300d8dfSJoonsoo Kim migrate_reason_names[page_owner->last_migrate_reason]); 503b1d5488aSCharan Teja Kalla page_ext_put(page_ext); 5044e462112SVlastimil Babka } 5054e462112SVlastimil Babka 50648c96a36SJoonsoo Kim static ssize_t 50748c96a36SJoonsoo Kim read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos) 50848c96a36SJoonsoo Kim { 50948c96a36SJoonsoo Kim unsigned long pfn; 51048c96a36SJoonsoo Kim struct page *page; 51148c96a36SJoonsoo Kim struct page_ext *page_ext; 5129300d8dfSJoonsoo Kim struct page_owner *page_owner; 513f2ca0b55SJoonsoo Kim depot_stack_handle_t handle; 51448c96a36SJoonsoo Kim 5157dd80b8aSVlastimil Babka if (!static_branch_unlikely(&page_owner_inited)) 51648c96a36SJoonsoo Kim return -EINVAL; 51748c96a36SJoonsoo Kim 51848c96a36SJoonsoo Kim page = NULL; 5198f0efa81SKassey Li if (*ppos == 0) 5208f0efa81SKassey Li pfn = min_low_pfn; 5218f0efa81SKassey Li else 5228f0efa81SKassey Li pfn = *ppos; 52348c96a36SJoonsoo Kim /* Find a valid PFN or the start of a MAX_ORDER_NR_PAGES area */ 52448c96a36SJoonsoo Kim while (!pfn_valid(pfn) && (pfn & (MAX_ORDER_NR_PAGES - 1)) != 0) 52548c96a36SJoonsoo Kim pfn++; 52648c96a36SJoonsoo Kim 52748c96a36SJoonsoo Kim /* Find an allocated page */ 52848c96a36SJoonsoo Kim for (; pfn < max_pfn; pfn++) { 52948c96a36SJoonsoo Kim /* 530b1d5488aSCharan Teja Kalla * This temporary page_owner is required so 531b1d5488aSCharan Teja Kalla * that we can avoid the context switches while holding 532b1d5488aSCharan Teja Kalla * the rcu lock and copying the page owner information to 533b1d5488aSCharan Teja Kalla * user through copy_to_user() or GFP_KERNEL allocations. 534b1d5488aSCharan Teja Kalla */ 535b1d5488aSCharan Teja Kalla struct page_owner page_owner_tmp; 536b1d5488aSCharan Teja Kalla 537b1d5488aSCharan Teja Kalla /* 53848c96a36SJoonsoo Kim * If the new page is in a new MAX_ORDER_NR_PAGES area, 53948c96a36SJoonsoo Kim * validate the area as existing, skip it if not 54048c96a36SJoonsoo Kim */ 54148c96a36SJoonsoo Kim if ((pfn & (MAX_ORDER_NR_PAGES - 1)) == 0 && !pfn_valid(pfn)) { 54248c96a36SJoonsoo Kim pfn += MAX_ORDER_NR_PAGES - 1; 54348c96a36SJoonsoo Kim continue; 54448c96a36SJoonsoo Kim } 54548c96a36SJoonsoo Kim 54648c96a36SJoonsoo Kim page = pfn_to_page(pfn); 54748c96a36SJoonsoo Kim if (PageBuddy(page)) { 548ab130f91SMatthew Wilcox (Oracle) unsigned long freepage_order = buddy_order_unsafe(page); 54948c96a36SJoonsoo Kim 55048c96a36SJoonsoo Kim if (freepage_order < MAX_ORDER) 55148c96a36SJoonsoo Kim pfn += (1UL << freepage_order) - 1; 55248c96a36SJoonsoo Kim continue; 55348c96a36SJoonsoo Kim } 55448c96a36SJoonsoo Kim 555b1d5488aSCharan Teja Kalla page_ext = page_ext_get(page); 556f86e4271SYang Shi if (unlikely(!page_ext)) 557f86e4271SYang Shi continue; 55848c96a36SJoonsoo Kim 55948c96a36SJoonsoo Kim /* 56061cf5febSJoonsoo Kim * Some pages could be missed by concurrent allocation or free, 56161cf5febSJoonsoo Kim * because we don't hold the zone lock. 56248c96a36SJoonsoo Kim */ 56348c96a36SJoonsoo Kim if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags)) 564b1d5488aSCharan Teja Kalla goto ext_put_continue; 56548c96a36SJoonsoo Kim 56637389167SVlastimil Babka /* 56737389167SVlastimil Babka * Although we do have the info about past allocation of free 56837389167SVlastimil Babka * pages, it's not relevant for current memory usage. 56937389167SVlastimil Babka */ 570fdf3bf80SVlastimil Babka if (!test_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags)) 571b1d5488aSCharan Teja Kalla goto ext_put_continue; 57237389167SVlastimil Babka 5739300d8dfSJoonsoo Kim page_owner = get_page_owner(page_ext); 5749300d8dfSJoonsoo Kim 575f2ca0b55SJoonsoo Kim /* 5767e2f2a0cSVlastimil Babka * Don't print "tail" pages of high-order allocations as that 5777e2f2a0cSVlastimil Babka * would inflate the stats. 5787e2f2a0cSVlastimil Babka */ 5797e2f2a0cSVlastimil Babka if (!IS_ALIGNED(pfn, 1 << page_owner->order)) 580b1d5488aSCharan Teja Kalla goto ext_put_continue; 5817e2f2a0cSVlastimil Babka 5827e2f2a0cSVlastimil Babka /* 583f2ca0b55SJoonsoo Kim * Access to page_ext->handle isn't synchronous so we should 584f2ca0b55SJoonsoo Kim * be careful to access it. 585f2ca0b55SJoonsoo Kim */ 5869300d8dfSJoonsoo Kim handle = READ_ONCE(page_owner->handle); 587f2ca0b55SJoonsoo Kim if (!handle) 588b1d5488aSCharan Teja Kalla goto ext_put_continue; 589f2ca0b55SJoonsoo Kim 59048c96a36SJoonsoo Kim /* Record the next PFN to read in the file offset */ 5918f0efa81SKassey Li *ppos = pfn + 1; 59248c96a36SJoonsoo Kim 593b1d5488aSCharan Teja Kalla page_owner_tmp = *page_owner; 594b1d5488aSCharan Teja Kalla page_ext_put(page_ext); 595f2ca0b55SJoonsoo Kim return print_page_owner(buf, count, pfn, page, 596b1d5488aSCharan Teja Kalla &page_owner_tmp, handle); 597b1d5488aSCharan Teja Kalla ext_put_continue: 598b1d5488aSCharan Teja Kalla page_ext_put(page_ext); 59948c96a36SJoonsoo Kim } 60048c96a36SJoonsoo Kim 60148c96a36SJoonsoo Kim return 0; 60248c96a36SJoonsoo Kim } 60348c96a36SJoonsoo Kim 6048f0efa81SKassey Li static loff_t lseek_page_owner(struct file *file, loff_t offset, int orig) 6058f0efa81SKassey Li { 6068f0efa81SKassey Li switch (orig) { 6078f0efa81SKassey Li case SEEK_SET: 6088f0efa81SKassey Li file->f_pos = offset; 6098f0efa81SKassey Li break; 6108f0efa81SKassey Li case SEEK_CUR: 6118f0efa81SKassey Li file->f_pos += offset; 6128f0efa81SKassey Li break; 6138f0efa81SKassey Li default: 6148f0efa81SKassey Li return -EINVAL; 6158f0efa81SKassey Li } 6168f0efa81SKassey Li return file->f_pos; 6178f0efa81SKassey Li } 6188f0efa81SKassey Li 61961cf5febSJoonsoo Kim static void init_pages_in_zone(pg_data_t *pgdat, struct zone *zone) 62061cf5febSJoonsoo Kim { 6216787c1daSOscar Salvador unsigned long pfn = zone->zone_start_pfn; 6226787c1daSOscar Salvador unsigned long end_pfn = zone_end_pfn(zone); 62361cf5febSJoonsoo Kim unsigned long count = 0; 62461cf5febSJoonsoo Kim 62561cf5febSJoonsoo Kim /* 62661cf5febSJoonsoo Kim * Walk the zone in pageblock_nr_pages steps. If a page block spans 62761cf5febSJoonsoo Kim * a zone boundary, it will be double counted between zones. This does 62861cf5febSJoonsoo Kim * not matter as the mixed block count will still be correct 62961cf5febSJoonsoo Kim */ 63061cf5febSJoonsoo Kim for (; pfn < end_pfn; ) { 6316787c1daSOscar Salvador unsigned long block_end_pfn; 6326787c1daSOscar Salvador 63361cf5febSJoonsoo Kim if (!pfn_valid(pfn)) { 63461cf5febSJoonsoo Kim pfn = ALIGN(pfn + 1, MAX_ORDER_NR_PAGES); 63561cf5febSJoonsoo Kim continue; 63661cf5febSJoonsoo Kim } 63761cf5febSJoonsoo Kim 638*4f9bc69aSKefeng Wang block_end_pfn = pageblock_end_pfn(pfn); 63961cf5febSJoonsoo Kim block_end_pfn = min(block_end_pfn, end_pfn); 64061cf5febSJoonsoo Kim 64161cf5febSJoonsoo Kim for (; pfn < block_end_pfn; pfn++) { 642859a85ddSMike Rapoport struct page *page = pfn_to_page(pfn); 6436787c1daSOscar Salvador struct page_ext *page_ext; 6446787c1daSOscar Salvador 6459d43f5aeSJoonsoo Kim if (page_zone(page) != zone) 6469d43f5aeSJoonsoo Kim continue; 6479d43f5aeSJoonsoo Kim 64861cf5febSJoonsoo Kim /* 64910903027SVlastimil Babka * To avoid having to grab zone->lock, be a little 65010903027SVlastimil Babka * careful when reading buddy page order. The only 65110903027SVlastimil Babka * danger is that we skip too much and potentially miss 65210903027SVlastimil Babka * some early allocated pages, which is better than 65310903027SVlastimil Babka * heavy lock contention. 65461cf5febSJoonsoo Kim */ 65561cf5febSJoonsoo Kim if (PageBuddy(page)) { 656ab130f91SMatthew Wilcox (Oracle) unsigned long order = buddy_order_unsafe(page); 65710903027SVlastimil Babka 65810903027SVlastimil Babka if (order > 0 && order < MAX_ORDER) 65910903027SVlastimil Babka pfn += (1UL << order) - 1; 66061cf5febSJoonsoo Kim continue; 66161cf5febSJoonsoo Kim } 66261cf5febSJoonsoo Kim 66361cf5febSJoonsoo Kim if (PageReserved(page)) 66461cf5febSJoonsoo Kim continue; 66561cf5febSJoonsoo Kim 666b1d5488aSCharan Teja Kalla page_ext = page_ext_get(page); 667f86e4271SYang Shi if (unlikely(!page_ext)) 668f86e4271SYang Shi continue; 66961cf5febSJoonsoo Kim 670dab4ead1SVlastimil Babka /* Maybe overlapping zone */ 67161cf5febSJoonsoo Kim if (test_bit(PAGE_EXT_OWNER, &page_ext->flags)) 672b1d5488aSCharan Teja Kalla goto ext_put_continue; 67361cf5febSJoonsoo Kim 67461cf5febSJoonsoo Kim /* Found early allocated page */ 67564ea78d2Szhongjiang-ali __set_page_owner_handle(page_ext, early_handle, 6767e2f2a0cSVlastimil Babka 0, 0); 67761cf5febSJoonsoo Kim count++; 678b1d5488aSCharan Teja Kalla ext_put_continue: 679b1d5488aSCharan Teja Kalla page_ext_put(page_ext); 68061cf5febSJoonsoo Kim } 68110903027SVlastimil Babka cond_resched(); 68261cf5febSJoonsoo Kim } 68361cf5febSJoonsoo Kim 68461cf5febSJoonsoo Kim pr_info("Node %d, zone %8s: page owner found early allocated %lu pages\n", 68561cf5febSJoonsoo Kim pgdat->node_id, zone->name, count); 68661cf5febSJoonsoo Kim } 68761cf5febSJoonsoo Kim 68861cf5febSJoonsoo Kim static void init_zones_in_node(pg_data_t *pgdat) 68961cf5febSJoonsoo Kim { 69061cf5febSJoonsoo Kim struct zone *zone; 69161cf5febSJoonsoo Kim struct zone *node_zones = pgdat->node_zones; 69261cf5febSJoonsoo Kim 69361cf5febSJoonsoo Kim for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) { 69461cf5febSJoonsoo Kim if (!populated_zone(zone)) 69561cf5febSJoonsoo Kim continue; 69661cf5febSJoonsoo Kim 69761cf5febSJoonsoo Kim init_pages_in_zone(pgdat, zone); 69861cf5febSJoonsoo Kim } 69961cf5febSJoonsoo Kim } 70061cf5febSJoonsoo Kim 70161cf5febSJoonsoo Kim static void init_early_allocated_pages(void) 70261cf5febSJoonsoo Kim { 70361cf5febSJoonsoo Kim pg_data_t *pgdat; 70461cf5febSJoonsoo Kim 70561cf5febSJoonsoo Kim for_each_online_pgdat(pgdat) 70661cf5febSJoonsoo Kim init_zones_in_node(pgdat); 70761cf5febSJoonsoo Kim } 70861cf5febSJoonsoo Kim 70948c96a36SJoonsoo Kim static const struct file_operations proc_page_owner_operations = { 71048c96a36SJoonsoo Kim .read = read_page_owner, 7118f0efa81SKassey Li .llseek = lseek_page_owner, 71248c96a36SJoonsoo Kim }; 71348c96a36SJoonsoo Kim 71448c96a36SJoonsoo Kim static int __init pageowner_init(void) 71548c96a36SJoonsoo Kim { 7167dd80b8aSVlastimil Babka if (!static_branch_unlikely(&page_owner_inited)) { 71748c96a36SJoonsoo Kim pr_info("page_owner is disabled\n"); 71848c96a36SJoonsoo Kim return 0; 71948c96a36SJoonsoo Kim } 72048c96a36SJoonsoo Kim 721d9f7979cSGreg Kroah-Hartman debugfs_create_file("page_owner", 0400, NULL, NULL, 722d9f7979cSGreg Kroah-Hartman &proc_page_owner_operations); 72348c96a36SJoonsoo Kim 724d9f7979cSGreg Kroah-Hartman return 0; 72548c96a36SJoonsoo Kim } 72644c5af96SPaul Gortmaker late_initcall(pageowner_init) 727