1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 248c96a36SJoonsoo Kim #include <linux/debugfs.h> 348c96a36SJoonsoo Kim #include <linux/mm.h> 448c96a36SJoonsoo Kim #include <linux/slab.h> 548c96a36SJoonsoo Kim #include <linux/uaccess.h> 657c8a661SMike Rapoport #include <linux/memblock.h> 748c96a36SJoonsoo Kim #include <linux/stacktrace.h> 848c96a36SJoonsoo Kim #include <linux/page_owner.h> 97dd80b8aSVlastimil Babka #include <linux/jump_label.h> 107cd12b4aSVlastimil Babka #include <linux/migrate.h> 11f2ca0b55SJoonsoo Kim #include <linux/stackdepot.h> 12e2f612e6SJoonsoo Kim #include <linux/seq_file.h> 13fcf89358SWaiman Long #include <linux/memcontrol.h> 149cc7e96aSLiam Mark #include <linux/sched/clock.h> 15f2ca0b55SJoonsoo Kim 1648c96a36SJoonsoo Kim #include "internal.h" 1748c96a36SJoonsoo Kim 18f2ca0b55SJoonsoo Kim /* 19f2ca0b55SJoonsoo Kim * TODO: teach PAGE_OWNER_STACK_DEPTH (__dump_page_owner and save_stack) 20f2ca0b55SJoonsoo Kim * to use off stack temporal storage 21f2ca0b55SJoonsoo Kim */ 22f2ca0b55SJoonsoo Kim #define PAGE_OWNER_STACK_DEPTH (16) 23f2ca0b55SJoonsoo Kim 249300d8dfSJoonsoo Kim struct page_owner { 256b4c54e3SAyush Mittal unsigned short order; 266b4c54e3SAyush Mittal short last_migrate_reason; 279300d8dfSJoonsoo Kim gfp_t gfp_mask; 289300d8dfSJoonsoo Kim depot_stack_handle_t handle; 298974558fSVlastimil Babka depot_stack_handle_t free_handle; 309cc7e96aSLiam Mark u64 ts_nsec; 31866b4852SGeorgi Djakov u64 free_ts_nsec; 32865ed6a3SWaiman Long char comm[TASK_COMM_LEN]; 339cc7e96aSLiam Mark pid_t pid; 34bf215eabSYixuan Cao pid_t tgid; 359300d8dfSJoonsoo Kim }; 369300d8dfSJoonsoo Kim 373645b5ecSFanjun Kong static bool page_owner_enabled __initdata; 387dd80b8aSVlastimil Babka DEFINE_STATIC_KEY_FALSE(page_owner_inited); 3948c96a36SJoonsoo Kim 40f2ca0b55SJoonsoo Kim static depot_stack_handle_t dummy_handle; 41f2ca0b55SJoonsoo Kim static depot_stack_handle_t failure_handle; 42dab4ead1SVlastimil Babka static depot_stack_handle_t early_handle; 43f2ca0b55SJoonsoo Kim 4461cf5febSJoonsoo Kim static void init_early_allocated_pages(void); 4561cf5febSJoonsoo Kim 461173194eSDou Liyang static int __init early_page_owner_param(char *buf) 4748c96a36SJoonsoo Kim { 48a5f1783bSVlastimil Babka int ret = kstrtobool(buf, &page_owner_enabled); 49a5f1783bSVlastimil Babka 50a5f1783bSVlastimil Babka if (page_owner_enabled) 511c0310adSAndrey Konovalov stack_depot_request_early_init(); 52a5f1783bSVlastimil Babka 53a5f1783bSVlastimil Babka return ret; 5448c96a36SJoonsoo Kim } 5548c96a36SJoonsoo Kim early_param("page_owner", early_page_owner_param); 5648c96a36SJoonsoo Kim 57cab0a7c1STing Liu static __init bool need_page_owner(void) 5848c96a36SJoonsoo Kim { 590fe9a448SVlastimil Babka return page_owner_enabled; 6048c96a36SJoonsoo Kim } 6148c96a36SJoonsoo Kim 62dab4ead1SVlastimil Babka static __always_inline depot_stack_handle_t create_dummy_stack(void) 63f2ca0b55SJoonsoo Kim { 64f2ca0b55SJoonsoo Kim unsigned long entries[4]; 65af52bf6bSThomas Gleixner unsigned int nr_entries; 66f2ca0b55SJoonsoo Kim 67af52bf6bSThomas Gleixner nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 0); 68af52bf6bSThomas Gleixner return stack_depot_save(entries, nr_entries, GFP_KERNEL); 69dab4ead1SVlastimil Babka } 70dab4ead1SVlastimil Babka 71dab4ead1SVlastimil Babka static noinline void register_dummy_stack(void) 72dab4ead1SVlastimil Babka { 73dab4ead1SVlastimil Babka dummy_handle = create_dummy_stack(); 74f2ca0b55SJoonsoo Kim } 75f2ca0b55SJoonsoo Kim 76f2ca0b55SJoonsoo Kim static noinline void register_failure_stack(void) 77f2ca0b55SJoonsoo Kim { 78dab4ead1SVlastimil Babka failure_handle = create_dummy_stack(); 79dab4ead1SVlastimil Babka } 80f2ca0b55SJoonsoo Kim 81dab4ead1SVlastimil Babka static noinline void register_early_stack(void) 82dab4ead1SVlastimil Babka { 83dab4ead1SVlastimil Babka early_handle = create_dummy_stack(); 84f2ca0b55SJoonsoo Kim } 85f2ca0b55SJoonsoo Kim 86cab0a7c1STing Liu static __init void init_page_owner(void) 8748c96a36SJoonsoo Kim { 880fe9a448SVlastimil Babka if (!page_owner_enabled) 8948c96a36SJoonsoo Kim return; 9048c96a36SJoonsoo Kim 91f2ca0b55SJoonsoo Kim register_dummy_stack(); 92f2ca0b55SJoonsoo Kim register_failure_stack(); 93dab4ead1SVlastimil Babka register_early_stack(); 947dd80b8aSVlastimil Babka static_branch_enable(&page_owner_inited); 9561cf5febSJoonsoo Kim init_early_allocated_pages(); 9648c96a36SJoonsoo Kim } 9748c96a36SJoonsoo Kim 9848c96a36SJoonsoo Kim struct page_ext_operations page_owner_ops = { 999300d8dfSJoonsoo Kim .size = sizeof(struct page_owner), 10048c96a36SJoonsoo Kim .need = need_page_owner, 10148c96a36SJoonsoo Kim .init = init_page_owner, 1026189eb82SPasha Tatashin .need_shared_flags = true, 10348c96a36SJoonsoo Kim }; 10448c96a36SJoonsoo Kim 1059300d8dfSJoonsoo Kim static inline struct page_owner *get_page_owner(struct page_ext *page_ext) 1069300d8dfSJoonsoo Kim { 1079300d8dfSJoonsoo Kim return (void *)page_ext + page_owner_ops.offset; 1089300d8dfSJoonsoo Kim } 1099300d8dfSJoonsoo Kim 110f2ca0b55SJoonsoo Kim static noinline depot_stack_handle_t save_stack(gfp_t flags) 111f2ca0b55SJoonsoo Kim { 112f2ca0b55SJoonsoo Kim unsigned long entries[PAGE_OWNER_STACK_DEPTH]; 113f2ca0b55SJoonsoo Kim depot_stack_handle_t handle; 114af52bf6bSThomas Gleixner unsigned int nr_entries; 115f2ca0b55SJoonsoo Kim 116f2ca0b55SJoonsoo Kim /* 1178e9b16c4SSergei Trofimovich * Avoid recursion. 1188e9b16c4SSergei Trofimovich * 1198e9b16c4SSergei Trofimovich * Sometimes page metadata allocation tracking requires more 1208e9b16c4SSergei Trofimovich * memory to be allocated: 1218e9b16c4SSergei Trofimovich * - when new stack trace is saved to stack depot 1228e9b16c4SSergei Trofimovich * - when backtrace itself is calculated (ia64) 123f2ca0b55SJoonsoo Kim */ 1248e9b16c4SSergei Trofimovich if (current->in_page_owner) 125f2ca0b55SJoonsoo Kim return dummy_handle; 1268e9b16c4SSergei Trofimovich current->in_page_owner = 1; 127f2ca0b55SJoonsoo Kim 1288e9b16c4SSergei Trofimovich nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 2); 129af52bf6bSThomas Gleixner handle = stack_depot_save(entries, nr_entries, flags); 130f2ca0b55SJoonsoo Kim if (!handle) 131f2ca0b55SJoonsoo Kim handle = failure_handle; 132f2ca0b55SJoonsoo Kim 1338e9b16c4SSergei Trofimovich current->in_page_owner = 0; 134f2ca0b55SJoonsoo Kim return handle; 135f2ca0b55SJoonsoo Kim } 136f2ca0b55SJoonsoo Kim 1370093de69SYixuan Cao void __reset_page_owner(struct page *page, unsigned short order) 1388974558fSVlastimil Babka { 1398974558fSVlastimil Babka int i; 1408974558fSVlastimil Babka struct page_ext *page_ext; 141fab765c2SSergei Trofimovich depot_stack_handle_t handle; 1428974558fSVlastimil Babka struct page_owner *page_owner; 143866b4852SGeorgi Djakov u64 free_ts_nsec = local_clock(); 1448974558fSVlastimil Babka 145b1d5488aSCharan Teja Kalla page_ext = page_ext_get(page); 1468974558fSVlastimil Babka if (unlikely(!page_ext)) 1475556cfe8SVlastimil Babka return; 148fab765c2SSergei Trofimovich 149fab765c2SSergei Trofimovich handle = save_stack(GFP_NOWAIT | __GFP_NOWARN); 1505556cfe8SVlastimil Babka for (i = 0; i < (1 << order); i++) { 151fdf3bf80SVlastimil Babka __clear_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags); 1528974558fSVlastimil Babka page_owner = get_page_owner(page_ext); 1538974558fSVlastimil Babka page_owner->free_handle = handle; 154866b4852SGeorgi Djakov page_owner->free_ts_nsec = free_ts_nsec; 1555556cfe8SVlastimil Babka page_ext = page_ext_next(page_ext); 1568974558fSVlastimil Babka } 157b1d5488aSCharan Teja Kalla page_ext_put(page_ext); 1588974558fSVlastimil Babka } 1598974558fSVlastimil Babka 16064ea78d2Szhongjiang-ali static inline void __set_page_owner_handle(struct page_ext *page_ext, 16164ea78d2Szhongjiang-ali depot_stack_handle_t handle, 1620093de69SYixuan Cao unsigned short order, gfp_t gfp_mask) 163f2ca0b55SJoonsoo Kim { 1649300d8dfSJoonsoo Kim struct page_owner *page_owner; 1657e2f2a0cSVlastimil Babka int i; 16605a42199SHyeonggon Yoo u64 ts_nsec = local_clock(); 16748c96a36SJoonsoo Kim 1687e2f2a0cSVlastimil Babka for (i = 0; i < (1 << order); i++) { 1699300d8dfSJoonsoo Kim page_owner = get_page_owner(page_ext); 170dab4ead1SVlastimil Babka page_owner->handle = handle; 1719300d8dfSJoonsoo Kim page_owner->order = order; 1729300d8dfSJoonsoo Kim page_owner->gfp_mask = gfp_mask; 1739300d8dfSJoonsoo Kim page_owner->last_migrate_reason = -1; 1749cc7e96aSLiam Mark page_owner->pid = current->pid; 175bf215eabSYixuan Cao page_owner->tgid = current->tgid; 17605a42199SHyeonggon Yoo page_owner->ts_nsec = ts_nsec; 177cd8c1fd8SEric Dumazet strscpy(page_owner->comm, current->comm, 178865ed6a3SWaiman Long sizeof(page_owner->comm)); 17948c96a36SJoonsoo Kim __set_bit(PAGE_EXT_OWNER, &page_ext->flags); 180fdf3bf80SVlastimil Babka __set_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags); 1817e2f2a0cSVlastimil Babka 1825556cfe8SVlastimil Babka page_ext = page_ext_next(page_ext); 1837e2f2a0cSVlastimil Babka } 18448c96a36SJoonsoo Kim } 18548c96a36SJoonsoo Kim 1860093de69SYixuan Cao noinline void __set_page_owner(struct page *page, unsigned short order, 187dab4ead1SVlastimil Babka gfp_t gfp_mask) 188dab4ead1SVlastimil Babka { 189b1d5488aSCharan Teja Kalla struct page_ext *page_ext; 190dab4ead1SVlastimil Babka depot_stack_handle_t handle; 191dab4ead1SVlastimil Babka 192b1d5488aSCharan Teja Kalla handle = save_stack(gfp_mask); 193b1d5488aSCharan Teja Kalla 194b1d5488aSCharan Teja Kalla page_ext = page_ext_get(page); 195dab4ead1SVlastimil Babka if (unlikely(!page_ext)) 196dab4ead1SVlastimil Babka return; 19764ea78d2Szhongjiang-ali __set_page_owner_handle(page_ext, handle, order, gfp_mask); 198b1d5488aSCharan Teja Kalla page_ext_put(page_ext); 199dab4ead1SVlastimil Babka } 200dab4ead1SVlastimil Babka 2017cd12b4aSVlastimil Babka void __set_page_owner_migrate_reason(struct page *page, int reason) 2027cd12b4aSVlastimil Babka { 203b1d5488aSCharan Teja Kalla struct page_ext *page_ext = page_ext_get(page); 2049300d8dfSJoonsoo Kim struct page_owner *page_owner; 2059300d8dfSJoonsoo Kim 206f86e4271SYang Shi if (unlikely(!page_ext)) 207f86e4271SYang Shi return; 2087cd12b4aSVlastimil Babka 2099300d8dfSJoonsoo Kim page_owner = get_page_owner(page_ext); 2109300d8dfSJoonsoo Kim page_owner->last_migrate_reason = reason; 211b1d5488aSCharan Teja Kalla page_ext_put(page_ext); 2127cd12b4aSVlastimil Babka } 2137cd12b4aSVlastimil Babka 2148fb156c9SMatthew Wilcox (Oracle) void __split_page_owner(struct page *page, unsigned int nr) 215e2cfc911SJoonsoo Kim { 216a9627bc5SJoonsoo Kim int i; 217b1d5488aSCharan Teja Kalla struct page_ext *page_ext = page_ext_get(page); 2189300d8dfSJoonsoo Kim struct page_owner *page_owner; 219e2cfc911SJoonsoo Kim 220a9627bc5SJoonsoo Kim if (unlikely(!page_ext)) 221a9627bc5SJoonsoo Kim return; 222a9627bc5SJoonsoo Kim 2238fb156c9SMatthew Wilcox (Oracle) for (i = 0; i < nr; i++) { 2249300d8dfSJoonsoo Kim page_owner = get_page_owner(page_ext); 2259300d8dfSJoonsoo Kim page_owner->order = 0; 2265556cfe8SVlastimil Babka page_ext = page_ext_next(page_ext); 2277e2f2a0cSVlastimil Babka } 228b1d5488aSCharan Teja Kalla page_ext_put(page_ext); 229e2cfc911SJoonsoo Kim } 230e2cfc911SJoonsoo Kim 23119138349SMatthew Wilcox (Oracle) void __folio_copy_owner(struct folio *newfolio, struct folio *old) 232d435edcaSVlastimil Babka { 233b1d5488aSCharan Teja Kalla struct page_ext *old_ext; 234b1d5488aSCharan Teja Kalla struct page_ext *new_ext; 2359300d8dfSJoonsoo Kim struct page_owner *old_page_owner, *new_page_owner; 236d435edcaSVlastimil Babka 237b1d5488aSCharan Teja Kalla old_ext = page_ext_get(&old->page); 238b1d5488aSCharan Teja Kalla if (unlikely(!old_ext)) 239f86e4271SYang Shi return; 240f86e4271SYang Shi 241b1d5488aSCharan Teja Kalla new_ext = page_ext_get(&newfolio->page); 242b1d5488aSCharan Teja Kalla if (unlikely(!new_ext)) { 243b1d5488aSCharan Teja Kalla page_ext_put(old_ext); 244b1d5488aSCharan Teja Kalla return; 245b1d5488aSCharan Teja Kalla } 246b1d5488aSCharan Teja Kalla 2479300d8dfSJoonsoo Kim old_page_owner = get_page_owner(old_ext); 2489300d8dfSJoonsoo Kim new_page_owner = get_page_owner(new_ext); 2499300d8dfSJoonsoo Kim new_page_owner->order = old_page_owner->order; 2509300d8dfSJoonsoo Kim new_page_owner->gfp_mask = old_page_owner->gfp_mask; 2519300d8dfSJoonsoo Kim new_page_owner->last_migrate_reason = 2529300d8dfSJoonsoo Kim old_page_owner->last_migrate_reason; 2539300d8dfSJoonsoo Kim new_page_owner->handle = old_page_owner->handle; 2549cc7e96aSLiam Mark new_page_owner->pid = old_page_owner->pid; 255bf215eabSYixuan Cao new_page_owner->tgid = old_page_owner->tgid; 2569cc7e96aSLiam Mark new_page_owner->ts_nsec = old_page_owner->ts_nsec; 257866b4852SGeorgi Djakov new_page_owner->free_ts_nsec = old_page_owner->ts_nsec; 258865ed6a3SWaiman Long strcpy(new_page_owner->comm, old_page_owner->comm); 259d435edcaSVlastimil Babka 260d435edcaSVlastimil Babka /* 26119138349SMatthew Wilcox (Oracle) * We don't clear the bit on the old folio as it's going to be freed 262d435edcaSVlastimil Babka * after migration. Until then, the info can be useful in case of 263f0953a1bSIngo Molnar * a bug, and the overall stats will be off a bit only temporarily. 264d435edcaSVlastimil Babka * Also, migrate_misplaced_transhuge_page() can still fail the 26519138349SMatthew Wilcox (Oracle) * migration and then we want the old folio to retain the info. But 266d435edcaSVlastimil Babka * in that case we also don't need to explicitly clear the info from 267d435edcaSVlastimil Babka * the new page, which will be freed. 268d435edcaSVlastimil Babka */ 269d435edcaSVlastimil Babka __set_bit(PAGE_EXT_OWNER, &new_ext->flags); 270fdf3bf80SVlastimil Babka __set_bit(PAGE_EXT_OWNER_ALLOCATED, &new_ext->flags); 271b1d5488aSCharan Teja Kalla page_ext_put(new_ext); 272b1d5488aSCharan Teja Kalla page_ext_put(old_ext); 273d435edcaSVlastimil Babka } 274d435edcaSVlastimil Babka 275e2f612e6SJoonsoo Kim void pagetypeinfo_showmixedcount_print(struct seq_file *m, 276e2f612e6SJoonsoo Kim pg_data_t *pgdat, struct zone *zone) 277e2f612e6SJoonsoo Kim { 278e2f612e6SJoonsoo Kim struct page *page; 279e2f612e6SJoonsoo Kim struct page_ext *page_ext; 2809300d8dfSJoonsoo Kim struct page_owner *page_owner; 2811d2cae8eSMiaohe Lin unsigned long pfn, block_end_pfn; 2821d2cae8eSMiaohe Lin unsigned long end_pfn = zone_end_pfn(zone); 283e2f612e6SJoonsoo Kim unsigned long count[MIGRATE_TYPES] = { 0, }; 284e2f612e6SJoonsoo Kim int pageblock_mt, page_mt; 285e2f612e6SJoonsoo Kim int i; 286e2f612e6SJoonsoo Kim 287e2f612e6SJoonsoo Kim /* Scan block by block. First and last block may be incomplete */ 288e2f612e6SJoonsoo Kim pfn = zone->zone_start_pfn; 289e2f612e6SJoonsoo Kim 290e2f612e6SJoonsoo Kim /* 291e2f612e6SJoonsoo Kim * Walk the zone in pageblock_nr_pages steps. If a page block spans 292e2f612e6SJoonsoo Kim * a zone boundary, it will be double counted between zones. This does 293e2f612e6SJoonsoo Kim * not matter as the mixed block count will still be correct 294e2f612e6SJoonsoo Kim */ 295e2f612e6SJoonsoo Kim for (; pfn < end_pfn; ) { 296a26ee565SQian Cai page = pfn_to_online_page(pfn); 297a26ee565SQian Cai if (!page) { 298e2f612e6SJoonsoo Kim pfn = ALIGN(pfn + 1, MAX_ORDER_NR_PAGES); 299e2f612e6SJoonsoo Kim continue; 300e2f612e6SJoonsoo Kim } 301e2f612e6SJoonsoo Kim 3024f9bc69aSKefeng Wang block_end_pfn = pageblock_end_pfn(pfn); 303e2f612e6SJoonsoo Kim block_end_pfn = min(block_end_pfn, end_pfn); 304e2f612e6SJoonsoo Kim 305e2f612e6SJoonsoo Kim pageblock_mt = get_pageblock_migratetype(page); 306e2f612e6SJoonsoo Kim 307e2f612e6SJoonsoo Kim for (; pfn < block_end_pfn; pfn++) { 308a26ee565SQian Cai /* The pageblock is online, no need to recheck. */ 309e2f612e6SJoonsoo Kim page = pfn_to_page(pfn); 310e2f612e6SJoonsoo Kim 311e2f612e6SJoonsoo Kim if (page_zone(page) != zone) 312e2f612e6SJoonsoo Kim continue; 313e2f612e6SJoonsoo Kim 314e2f612e6SJoonsoo Kim if (PageBuddy(page)) { 315727c080fSVinayak Menon unsigned long freepage_order; 316727c080fSVinayak Menon 317ab130f91SMatthew Wilcox (Oracle) freepage_order = buddy_order_unsafe(page); 31823baf831SKirill A. Shutemov if (freepage_order <= MAX_ORDER) 319727c080fSVinayak Menon pfn += (1UL << freepage_order) - 1; 320e2f612e6SJoonsoo Kim continue; 321e2f612e6SJoonsoo Kim } 322e2f612e6SJoonsoo Kim 323e2f612e6SJoonsoo Kim if (PageReserved(page)) 324e2f612e6SJoonsoo Kim continue; 325e2f612e6SJoonsoo Kim 326b1d5488aSCharan Teja Kalla page_ext = page_ext_get(page); 327e2f612e6SJoonsoo Kim if (unlikely(!page_ext)) 328e2f612e6SJoonsoo Kim continue; 329e2f612e6SJoonsoo Kim 330fdf3bf80SVlastimil Babka if (!test_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags)) 331b1d5488aSCharan Teja Kalla goto ext_put_continue; 332e2f612e6SJoonsoo Kim 3339300d8dfSJoonsoo Kim page_owner = get_page_owner(page_ext); 33401c0bfe0SWei Yang page_mt = gfp_migratetype(page_owner->gfp_mask); 335e2f612e6SJoonsoo Kim if (pageblock_mt != page_mt) { 336e2f612e6SJoonsoo Kim if (is_migrate_cma(pageblock_mt)) 337e2f612e6SJoonsoo Kim count[MIGRATE_MOVABLE]++; 338e2f612e6SJoonsoo Kim else 339e2f612e6SJoonsoo Kim count[pageblock_mt]++; 340e2f612e6SJoonsoo Kim 341e2f612e6SJoonsoo Kim pfn = block_end_pfn; 342b1d5488aSCharan Teja Kalla page_ext_put(page_ext); 343e2f612e6SJoonsoo Kim break; 344e2f612e6SJoonsoo Kim } 3459300d8dfSJoonsoo Kim pfn += (1UL << page_owner->order) - 1; 346b1d5488aSCharan Teja Kalla ext_put_continue: 347b1d5488aSCharan Teja Kalla page_ext_put(page_ext); 348e2f612e6SJoonsoo Kim } 349e2f612e6SJoonsoo Kim } 350e2f612e6SJoonsoo Kim 351e2f612e6SJoonsoo Kim /* Print counts */ 352e2f612e6SJoonsoo Kim seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name); 353e2f612e6SJoonsoo Kim for (i = 0; i < MIGRATE_TYPES; i++) 354e2f612e6SJoonsoo Kim seq_printf(m, "%12lu ", count[i]); 355e2f612e6SJoonsoo Kim seq_putc(m, '\n'); 356e2f612e6SJoonsoo Kim } 357e2f612e6SJoonsoo Kim 358fcf89358SWaiman Long /* 359fcf89358SWaiman Long * Looking for memcg information and print it out 360fcf89358SWaiman Long */ 361fcf89358SWaiman Long static inline int print_page_owner_memcg(char *kbuf, size_t count, int ret, 362fcf89358SWaiman Long struct page *page) 363fcf89358SWaiman Long { 364fcf89358SWaiman Long #ifdef CONFIG_MEMCG 365fcf89358SWaiman Long unsigned long memcg_data; 366fcf89358SWaiman Long struct mem_cgroup *memcg; 367fcf89358SWaiman Long bool online; 368fcf89358SWaiman Long char name[80]; 369fcf89358SWaiman Long 370fcf89358SWaiman Long rcu_read_lock(); 371fcf89358SWaiman Long memcg_data = READ_ONCE(page->memcg_data); 372fcf89358SWaiman Long if (!memcg_data) 373fcf89358SWaiman Long goto out_unlock; 374fcf89358SWaiman Long 375fcf89358SWaiman Long if (memcg_data & MEMCG_DATA_OBJCGS) 376fcf89358SWaiman Long ret += scnprintf(kbuf + ret, count - ret, 377fcf89358SWaiman Long "Slab cache page\n"); 378fcf89358SWaiman Long 379fcf89358SWaiman Long memcg = page_memcg_check(page); 380fcf89358SWaiman Long if (!memcg) 381fcf89358SWaiman Long goto out_unlock; 382fcf89358SWaiman Long 383fcf89358SWaiman Long online = (memcg->css.flags & CSS_ONLINE); 384fcf89358SWaiman Long cgroup_name(memcg->css.cgroup, name, sizeof(name)); 385fcf89358SWaiman Long ret += scnprintf(kbuf + ret, count - ret, 386fcf89358SWaiman Long "Charged %sto %smemcg %s\n", 387fcf89358SWaiman Long PageMemcgKmem(page) ? "(via objcg) " : "", 388fcf89358SWaiman Long online ? "" : "offline ", 389fcf89358SWaiman Long name); 390fcf89358SWaiman Long out_unlock: 391fcf89358SWaiman Long rcu_read_unlock(); 392fcf89358SWaiman Long #endif /* CONFIG_MEMCG */ 393fcf89358SWaiman Long 394fcf89358SWaiman Long return ret; 395fcf89358SWaiman Long } 396fcf89358SWaiman Long 39748c96a36SJoonsoo Kim static ssize_t 39848c96a36SJoonsoo Kim print_page_owner(char __user *buf, size_t count, unsigned long pfn, 3999300d8dfSJoonsoo Kim struct page *page, struct page_owner *page_owner, 400f2ca0b55SJoonsoo Kim depot_stack_handle_t handle) 40148c96a36SJoonsoo Kim { 402af52bf6bSThomas Gleixner int ret, pageblock_mt, page_mt; 40348c96a36SJoonsoo Kim char *kbuf; 40448c96a36SJoonsoo Kim 405c8f61cfcSMiles Chen count = min_t(size_t, count, PAGE_SIZE); 40648c96a36SJoonsoo Kim kbuf = kmalloc(count, GFP_KERNEL); 40748c96a36SJoonsoo Kim if (!kbuf) 40848c96a36SJoonsoo Kim return -ENOMEM; 40948c96a36SJoonsoo Kim 4103ebc4397SWaiman Long ret = scnprintf(kbuf, count, 411bf215eabSYixuan Cao "Page allocated via order %u, mask %#x(%pGg), pid %d, tgid %d (%s), ts %llu ns, free_ts %llu ns\n", 4129300d8dfSJoonsoo Kim page_owner->order, page_owner->gfp_mask, 4139cc7e96aSLiam Mark &page_owner->gfp_mask, page_owner->pid, 414bf215eabSYixuan Cao page_owner->tgid, page_owner->comm, 415bf215eabSYixuan Cao page_owner->ts_nsec, page_owner->free_ts_nsec); 41648c96a36SJoonsoo Kim 41748c96a36SJoonsoo Kim /* Print information relevant to grouping pages by mobility */ 4180b423ca2SMel Gorman pageblock_mt = get_pageblock_migratetype(page); 41901c0bfe0SWei Yang page_mt = gfp_migratetype(page_owner->gfp_mask); 4203ebc4397SWaiman Long ret += scnprintf(kbuf + ret, count - ret, 421*399fd496SKassey Li "PFN 0x%lx type %s Block %lu type %s Flags %pGp\n", 42248c96a36SJoonsoo Kim pfn, 42360f30350SVlastimil Babka migratetype_names[page_mt], 42448c96a36SJoonsoo Kim pfn >> pageblock_order, 42560f30350SVlastimil Babka migratetype_names[pageblock_mt], 42623efd080SMatthew Wilcox (Oracle) &page->flags); 42748c96a36SJoonsoo Kim 4280f68d45eSImran Khan ret += stack_depot_snprint(handle, kbuf + ret, count - ret, 0); 42948c96a36SJoonsoo Kim if (ret >= count) 43048c96a36SJoonsoo Kim goto err; 43148c96a36SJoonsoo Kim 4329300d8dfSJoonsoo Kim if (page_owner->last_migrate_reason != -1) { 4333ebc4397SWaiman Long ret += scnprintf(kbuf + ret, count - ret, 4347cd12b4aSVlastimil Babka "Page has been migrated, last migrate reason: %s\n", 4359300d8dfSJoonsoo Kim migrate_reason_names[page_owner->last_migrate_reason]); 4367cd12b4aSVlastimil Babka } 4377cd12b4aSVlastimil Babka 438fcf89358SWaiman Long ret = print_page_owner_memcg(kbuf, count, ret, page); 439fcf89358SWaiman Long 44048c96a36SJoonsoo Kim ret += snprintf(kbuf + ret, count - ret, "\n"); 44148c96a36SJoonsoo Kim if (ret >= count) 44248c96a36SJoonsoo Kim goto err; 44348c96a36SJoonsoo Kim 44448c96a36SJoonsoo Kim if (copy_to_user(buf, kbuf, ret)) 44548c96a36SJoonsoo Kim ret = -EFAULT; 44648c96a36SJoonsoo Kim 44748c96a36SJoonsoo Kim kfree(kbuf); 44848c96a36SJoonsoo Kim return ret; 44948c96a36SJoonsoo Kim 45048c96a36SJoonsoo Kim err: 45148c96a36SJoonsoo Kim kfree(kbuf); 45248c96a36SJoonsoo Kim return -ENOMEM; 45348c96a36SJoonsoo Kim } 45448c96a36SJoonsoo Kim 4558bf6f451SMatthew Wilcox (Oracle) void __dump_page_owner(const struct page *page) 4564e462112SVlastimil Babka { 457b1d5488aSCharan Teja Kalla struct page_ext *page_ext = page_ext_get((void *)page); 4589300d8dfSJoonsoo Kim struct page_owner *page_owner; 459f2ca0b55SJoonsoo Kim depot_stack_handle_t handle; 4608285027fSSudip Mukherjee gfp_t gfp_mask; 4618285027fSSudip Mukherjee int mt; 4624e462112SVlastimil Babka 463f86e4271SYang Shi if (unlikely(!page_ext)) { 464f86e4271SYang Shi pr_alert("There is not page extension available.\n"); 465f86e4271SYang Shi return; 466f86e4271SYang Shi } 4679300d8dfSJoonsoo Kim 4689300d8dfSJoonsoo Kim page_owner = get_page_owner(page_ext); 4699300d8dfSJoonsoo Kim gfp_mask = page_owner->gfp_mask; 47001c0bfe0SWei Yang mt = gfp_migratetype(gfp_mask); 471f86e4271SYang Shi 4724e462112SVlastimil Babka if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags)) { 47337389167SVlastimil Babka pr_alert("page_owner info is not present (never set?)\n"); 474b1d5488aSCharan Teja Kalla page_ext_put(page_ext); 4754e462112SVlastimil Babka return; 4764e462112SVlastimil Babka } 4774e462112SVlastimil Babka 478fdf3bf80SVlastimil Babka if (test_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags)) 47937389167SVlastimil Babka pr_alert("page_owner tracks the page as allocated\n"); 48037389167SVlastimil Babka else 48137389167SVlastimil Babka pr_alert("page_owner tracks the page as freed\n"); 48237389167SVlastimil Babka 483bf215eabSYixuan Cao pr_alert("page last allocated via order %u, migratetype %s, gfp_mask %#x(%pGg), pid %d, tgid %d (%s), ts %llu, free_ts %llu\n", 4849cc7e96aSLiam Mark page_owner->order, migratetype_names[mt], gfp_mask, &gfp_mask, 485bf215eabSYixuan Cao page_owner->pid, page_owner->tgid, page_owner->comm, 486bf215eabSYixuan Cao page_owner->ts_nsec, page_owner->free_ts_nsec); 48737389167SVlastimil Babka 4889300d8dfSJoonsoo Kim handle = READ_ONCE(page_owner->handle); 489505be481SImran Khan if (!handle) 49037389167SVlastimil Babka pr_alert("page_owner allocation stack trace missing\n"); 491505be481SImran Khan else 492505be481SImran Khan stack_depot_print(handle); 4934e462112SVlastimil Babka 4948974558fSVlastimil Babka handle = READ_ONCE(page_owner->free_handle); 4958974558fSVlastimil Babka if (!handle) { 4968974558fSVlastimil Babka pr_alert("page_owner free stack trace missing\n"); 4978974558fSVlastimil Babka } else { 4988974558fSVlastimil Babka pr_alert("page last free stack trace:\n"); 499505be481SImran Khan stack_depot_print(handle); 5008974558fSVlastimil Babka } 5018974558fSVlastimil Babka 5029300d8dfSJoonsoo Kim if (page_owner->last_migrate_reason != -1) 5034e462112SVlastimil Babka pr_alert("page has been migrated, last migrate reason: %s\n", 5049300d8dfSJoonsoo Kim migrate_reason_names[page_owner->last_migrate_reason]); 505b1d5488aSCharan Teja Kalla page_ext_put(page_ext); 5064e462112SVlastimil Babka } 5074e462112SVlastimil Babka 50848c96a36SJoonsoo Kim static ssize_t 50948c96a36SJoonsoo Kim read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos) 51048c96a36SJoonsoo Kim { 51148c96a36SJoonsoo Kim unsigned long pfn; 51248c96a36SJoonsoo Kim struct page *page; 51348c96a36SJoonsoo Kim struct page_ext *page_ext; 5149300d8dfSJoonsoo Kim struct page_owner *page_owner; 515f2ca0b55SJoonsoo Kim depot_stack_handle_t handle; 51648c96a36SJoonsoo Kim 5177dd80b8aSVlastimil Babka if (!static_branch_unlikely(&page_owner_inited)) 51848c96a36SJoonsoo Kim return -EINVAL; 51948c96a36SJoonsoo Kim 52048c96a36SJoonsoo Kim page = NULL; 5218f0efa81SKassey Li if (*ppos == 0) 5228f0efa81SKassey Li pfn = min_low_pfn; 5238f0efa81SKassey Li else 5248f0efa81SKassey Li pfn = *ppos; 52548c96a36SJoonsoo Kim /* Find a valid PFN or the start of a MAX_ORDER_NR_PAGES area */ 52648c96a36SJoonsoo Kim while (!pfn_valid(pfn) && (pfn & (MAX_ORDER_NR_PAGES - 1)) != 0) 52748c96a36SJoonsoo Kim pfn++; 52848c96a36SJoonsoo Kim 52948c96a36SJoonsoo Kim /* Find an allocated page */ 53048c96a36SJoonsoo Kim for (; pfn < max_pfn; pfn++) { 53148c96a36SJoonsoo Kim /* 532b1d5488aSCharan Teja Kalla * This temporary page_owner is required so 533b1d5488aSCharan Teja Kalla * that we can avoid the context switches while holding 534b1d5488aSCharan Teja Kalla * the rcu lock and copying the page owner information to 535b1d5488aSCharan Teja Kalla * user through copy_to_user() or GFP_KERNEL allocations. 536b1d5488aSCharan Teja Kalla */ 537b1d5488aSCharan Teja Kalla struct page_owner page_owner_tmp; 538b1d5488aSCharan Teja Kalla 539b1d5488aSCharan Teja Kalla /* 54048c96a36SJoonsoo Kim * If the new page is in a new MAX_ORDER_NR_PAGES area, 54148c96a36SJoonsoo Kim * validate the area as existing, skip it if not 54248c96a36SJoonsoo Kim */ 54348c96a36SJoonsoo Kim if ((pfn & (MAX_ORDER_NR_PAGES - 1)) == 0 && !pfn_valid(pfn)) { 54448c96a36SJoonsoo Kim pfn += MAX_ORDER_NR_PAGES - 1; 54548c96a36SJoonsoo Kim continue; 54648c96a36SJoonsoo Kim } 54748c96a36SJoonsoo Kim 54848c96a36SJoonsoo Kim page = pfn_to_page(pfn); 54948c96a36SJoonsoo Kim if (PageBuddy(page)) { 550ab130f91SMatthew Wilcox (Oracle) unsigned long freepage_order = buddy_order_unsafe(page); 55148c96a36SJoonsoo Kim 55223baf831SKirill A. Shutemov if (freepage_order <= MAX_ORDER) 55348c96a36SJoonsoo Kim pfn += (1UL << freepage_order) - 1; 55448c96a36SJoonsoo Kim continue; 55548c96a36SJoonsoo Kim } 55648c96a36SJoonsoo Kim 557b1d5488aSCharan Teja Kalla page_ext = page_ext_get(page); 558f86e4271SYang Shi if (unlikely(!page_ext)) 559f86e4271SYang Shi continue; 56048c96a36SJoonsoo Kim 56148c96a36SJoonsoo Kim /* 56261cf5febSJoonsoo Kim * Some pages could be missed by concurrent allocation or free, 56361cf5febSJoonsoo Kim * because we don't hold the zone lock. 56448c96a36SJoonsoo Kim */ 56548c96a36SJoonsoo Kim if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags)) 566b1d5488aSCharan Teja Kalla goto ext_put_continue; 56748c96a36SJoonsoo Kim 56837389167SVlastimil Babka /* 56937389167SVlastimil Babka * Although we do have the info about past allocation of free 57037389167SVlastimil Babka * pages, it's not relevant for current memory usage. 57137389167SVlastimil Babka */ 572fdf3bf80SVlastimil Babka if (!test_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags)) 573b1d5488aSCharan Teja Kalla goto ext_put_continue; 57437389167SVlastimil Babka 5759300d8dfSJoonsoo Kim page_owner = get_page_owner(page_ext); 5769300d8dfSJoonsoo Kim 577f2ca0b55SJoonsoo Kim /* 5787e2f2a0cSVlastimil Babka * Don't print "tail" pages of high-order allocations as that 5797e2f2a0cSVlastimil Babka * would inflate the stats. 5807e2f2a0cSVlastimil Babka */ 5817e2f2a0cSVlastimil Babka if (!IS_ALIGNED(pfn, 1 << page_owner->order)) 582b1d5488aSCharan Teja Kalla goto ext_put_continue; 5837e2f2a0cSVlastimil Babka 5847e2f2a0cSVlastimil Babka /* 585f2ca0b55SJoonsoo Kim * Access to page_ext->handle isn't synchronous so we should 586f2ca0b55SJoonsoo Kim * be careful to access it. 587f2ca0b55SJoonsoo Kim */ 5889300d8dfSJoonsoo Kim handle = READ_ONCE(page_owner->handle); 589f2ca0b55SJoonsoo Kim if (!handle) 590b1d5488aSCharan Teja Kalla goto ext_put_continue; 591f2ca0b55SJoonsoo Kim 59248c96a36SJoonsoo Kim /* Record the next PFN to read in the file offset */ 5938f0efa81SKassey Li *ppos = pfn + 1; 59448c96a36SJoonsoo Kim 595b1d5488aSCharan Teja Kalla page_owner_tmp = *page_owner; 596b1d5488aSCharan Teja Kalla page_ext_put(page_ext); 597f2ca0b55SJoonsoo Kim return print_page_owner(buf, count, pfn, page, 598b1d5488aSCharan Teja Kalla &page_owner_tmp, handle); 599b1d5488aSCharan Teja Kalla ext_put_continue: 600b1d5488aSCharan Teja Kalla page_ext_put(page_ext); 60148c96a36SJoonsoo Kim } 60248c96a36SJoonsoo Kim 60348c96a36SJoonsoo Kim return 0; 60448c96a36SJoonsoo Kim } 60548c96a36SJoonsoo Kim 6068f0efa81SKassey Li static loff_t lseek_page_owner(struct file *file, loff_t offset, int orig) 6078f0efa81SKassey Li { 6088f0efa81SKassey Li switch (orig) { 6098f0efa81SKassey Li case SEEK_SET: 6108f0efa81SKassey Li file->f_pos = offset; 6118f0efa81SKassey Li break; 6128f0efa81SKassey Li case SEEK_CUR: 6138f0efa81SKassey Li file->f_pos += offset; 6148f0efa81SKassey Li break; 6158f0efa81SKassey Li default: 6168f0efa81SKassey Li return -EINVAL; 6178f0efa81SKassey Li } 6188f0efa81SKassey Li return file->f_pos; 6198f0efa81SKassey Li } 6208f0efa81SKassey Li 62161cf5febSJoonsoo Kim static void init_pages_in_zone(pg_data_t *pgdat, struct zone *zone) 62261cf5febSJoonsoo Kim { 6236787c1daSOscar Salvador unsigned long pfn = zone->zone_start_pfn; 6246787c1daSOscar Salvador unsigned long end_pfn = zone_end_pfn(zone); 62561cf5febSJoonsoo Kim unsigned long count = 0; 62661cf5febSJoonsoo Kim 62761cf5febSJoonsoo Kim /* 62861cf5febSJoonsoo Kim * Walk the zone in pageblock_nr_pages steps. If a page block spans 62961cf5febSJoonsoo Kim * a zone boundary, it will be double counted between zones. This does 63061cf5febSJoonsoo Kim * not matter as the mixed block count will still be correct 63161cf5febSJoonsoo Kim */ 63261cf5febSJoonsoo Kim for (; pfn < end_pfn; ) { 6336787c1daSOscar Salvador unsigned long block_end_pfn; 6346787c1daSOscar Salvador 63561cf5febSJoonsoo Kim if (!pfn_valid(pfn)) { 63661cf5febSJoonsoo Kim pfn = ALIGN(pfn + 1, MAX_ORDER_NR_PAGES); 63761cf5febSJoonsoo Kim continue; 63861cf5febSJoonsoo Kim } 63961cf5febSJoonsoo Kim 6404f9bc69aSKefeng Wang block_end_pfn = pageblock_end_pfn(pfn); 64161cf5febSJoonsoo Kim block_end_pfn = min(block_end_pfn, end_pfn); 64261cf5febSJoonsoo Kim 64361cf5febSJoonsoo Kim for (; pfn < block_end_pfn; pfn++) { 644859a85ddSMike Rapoport struct page *page = pfn_to_page(pfn); 6456787c1daSOscar Salvador struct page_ext *page_ext; 6466787c1daSOscar Salvador 6479d43f5aeSJoonsoo Kim if (page_zone(page) != zone) 6489d43f5aeSJoonsoo Kim continue; 6499d43f5aeSJoonsoo Kim 65061cf5febSJoonsoo Kim /* 65110903027SVlastimil Babka * To avoid having to grab zone->lock, be a little 65210903027SVlastimil Babka * careful when reading buddy page order. The only 65310903027SVlastimil Babka * danger is that we skip too much and potentially miss 65410903027SVlastimil Babka * some early allocated pages, which is better than 65510903027SVlastimil Babka * heavy lock contention. 65661cf5febSJoonsoo Kim */ 65761cf5febSJoonsoo Kim if (PageBuddy(page)) { 658ab130f91SMatthew Wilcox (Oracle) unsigned long order = buddy_order_unsafe(page); 65910903027SVlastimil Babka 66023baf831SKirill A. Shutemov if (order > 0 && order <= MAX_ORDER) 66110903027SVlastimil Babka pfn += (1UL << order) - 1; 66261cf5febSJoonsoo Kim continue; 66361cf5febSJoonsoo Kim } 66461cf5febSJoonsoo Kim 66561cf5febSJoonsoo Kim if (PageReserved(page)) 66661cf5febSJoonsoo Kim continue; 66761cf5febSJoonsoo Kim 668b1d5488aSCharan Teja Kalla page_ext = page_ext_get(page); 669f86e4271SYang Shi if (unlikely(!page_ext)) 670f86e4271SYang Shi continue; 67161cf5febSJoonsoo Kim 672dab4ead1SVlastimil Babka /* Maybe overlapping zone */ 67361cf5febSJoonsoo Kim if (test_bit(PAGE_EXT_OWNER, &page_ext->flags)) 674b1d5488aSCharan Teja Kalla goto ext_put_continue; 67561cf5febSJoonsoo Kim 67661cf5febSJoonsoo Kim /* Found early allocated page */ 67764ea78d2Szhongjiang-ali __set_page_owner_handle(page_ext, early_handle, 6787e2f2a0cSVlastimil Babka 0, 0); 67961cf5febSJoonsoo Kim count++; 680b1d5488aSCharan Teja Kalla ext_put_continue: 681b1d5488aSCharan Teja Kalla page_ext_put(page_ext); 68261cf5febSJoonsoo Kim } 68310903027SVlastimil Babka cond_resched(); 68461cf5febSJoonsoo Kim } 68561cf5febSJoonsoo Kim 68661cf5febSJoonsoo Kim pr_info("Node %d, zone %8s: page owner found early allocated %lu pages\n", 68761cf5febSJoonsoo Kim pgdat->node_id, zone->name, count); 68861cf5febSJoonsoo Kim } 68961cf5febSJoonsoo Kim 69061cf5febSJoonsoo Kim static void init_zones_in_node(pg_data_t *pgdat) 69161cf5febSJoonsoo Kim { 69261cf5febSJoonsoo Kim struct zone *zone; 69361cf5febSJoonsoo Kim struct zone *node_zones = pgdat->node_zones; 69461cf5febSJoonsoo Kim 69561cf5febSJoonsoo Kim for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) { 69661cf5febSJoonsoo Kim if (!populated_zone(zone)) 69761cf5febSJoonsoo Kim continue; 69861cf5febSJoonsoo Kim 69961cf5febSJoonsoo Kim init_pages_in_zone(pgdat, zone); 70061cf5febSJoonsoo Kim } 70161cf5febSJoonsoo Kim } 70261cf5febSJoonsoo Kim 70361cf5febSJoonsoo Kim static void init_early_allocated_pages(void) 70461cf5febSJoonsoo Kim { 70561cf5febSJoonsoo Kim pg_data_t *pgdat; 70661cf5febSJoonsoo Kim 70761cf5febSJoonsoo Kim for_each_online_pgdat(pgdat) 70861cf5febSJoonsoo Kim init_zones_in_node(pgdat); 70961cf5febSJoonsoo Kim } 71061cf5febSJoonsoo Kim 71148c96a36SJoonsoo Kim static const struct file_operations proc_page_owner_operations = { 71248c96a36SJoonsoo Kim .read = read_page_owner, 7138f0efa81SKassey Li .llseek = lseek_page_owner, 71448c96a36SJoonsoo Kim }; 71548c96a36SJoonsoo Kim 71648c96a36SJoonsoo Kim static int __init pageowner_init(void) 71748c96a36SJoonsoo Kim { 7187dd80b8aSVlastimil Babka if (!static_branch_unlikely(&page_owner_inited)) { 71948c96a36SJoonsoo Kim pr_info("page_owner is disabled\n"); 72048c96a36SJoonsoo Kim return 0; 72148c96a36SJoonsoo Kim } 72248c96a36SJoonsoo Kim 723d9f7979cSGreg Kroah-Hartman debugfs_create_file("page_owner", 0400, NULL, NULL, 724d9f7979cSGreg Kroah-Hartman &proc_page_owner_operations); 72548c96a36SJoonsoo Kim 726d9f7979cSGreg Kroah-Hartman return 0; 72748c96a36SJoonsoo Kim } 72844c5af96SPaul Gortmaker late_initcall(pageowner_init) 729