1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 248c96a36SJoonsoo Kim #include <linux/debugfs.h> 348c96a36SJoonsoo Kim #include <linux/mm.h> 448c96a36SJoonsoo Kim #include <linux/slab.h> 548c96a36SJoonsoo Kim #include <linux/uaccess.h> 657c8a661SMike Rapoport #include <linux/memblock.h> 748c96a36SJoonsoo Kim #include <linux/stacktrace.h> 848c96a36SJoonsoo Kim #include <linux/page_owner.h> 97dd80b8aSVlastimil Babka #include <linux/jump_label.h> 107cd12b4aSVlastimil Babka #include <linux/migrate.h> 11f2ca0b55SJoonsoo Kim #include <linux/stackdepot.h> 12e2f612e6SJoonsoo Kim #include <linux/seq_file.h> 139cc7e96aSLiam Mark #include <linux/sched/clock.h> 14f2ca0b55SJoonsoo Kim 1548c96a36SJoonsoo Kim #include "internal.h" 1648c96a36SJoonsoo Kim 17f2ca0b55SJoonsoo Kim /* 18f2ca0b55SJoonsoo Kim * TODO: teach PAGE_OWNER_STACK_DEPTH (__dump_page_owner and save_stack) 19f2ca0b55SJoonsoo Kim * to use off stack temporal storage 20f2ca0b55SJoonsoo Kim */ 21f2ca0b55SJoonsoo Kim #define PAGE_OWNER_STACK_DEPTH (16) 22f2ca0b55SJoonsoo Kim 239300d8dfSJoonsoo Kim struct page_owner { 246b4c54e3SAyush Mittal unsigned short order; 256b4c54e3SAyush Mittal short last_migrate_reason; 269300d8dfSJoonsoo Kim gfp_t gfp_mask; 279300d8dfSJoonsoo Kim depot_stack_handle_t handle; 288974558fSVlastimil Babka depot_stack_handle_t free_handle; 299cc7e96aSLiam Mark u64 ts_nsec; 309cc7e96aSLiam Mark pid_t pid; 319300d8dfSJoonsoo Kim }; 329300d8dfSJoonsoo Kim 330fe9a448SVlastimil Babka static bool page_owner_enabled = false; 347dd80b8aSVlastimil Babka DEFINE_STATIC_KEY_FALSE(page_owner_inited); 3548c96a36SJoonsoo Kim 36f2ca0b55SJoonsoo Kim static depot_stack_handle_t dummy_handle; 37f2ca0b55SJoonsoo Kim static depot_stack_handle_t failure_handle; 38dab4ead1SVlastimil Babka static depot_stack_handle_t early_handle; 39f2ca0b55SJoonsoo Kim 4061cf5febSJoonsoo Kim static void init_early_allocated_pages(void); 4161cf5febSJoonsoo Kim 421173194eSDou Liyang static int __init early_page_owner_param(char *buf) 4348c96a36SJoonsoo Kim { 4448c96a36SJoonsoo Kim if (!buf) 4548c96a36SJoonsoo Kim return -EINVAL; 4648c96a36SJoonsoo Kim 4748c96a36SJoonsoo Kim if (strcmp(buf, "on") == 0) 480fe9a448SVlastimil Babka page_owner_enabled = true; 4948c96a36SJoonsoo Kim 5048c96a36SJoonsoo Kim return 0; 5148c96a36SJoonsoo Kim } 5248c96a36SJoonsoo Kim early_param("page_owner", early_page_owner_param); 5348c96a36SJoonsoo Kim 5448c96a36SJoonsoo Kim static bool need_page_owner(void) 5548c96a36SJoonsoo Kim { 560fe9a448SVlastimil Babka return page_owner_enabled; 5748c96a36SJoonsoo Kim } 5848c96a36SJoonsoo Kim 59dab4ead1SVlastimil Babka static __always_inline depot_stack_handle_t create_dummy_stack(void) 60f2ca0b55SJoonsoo Kim { 61f2ca0b55SJoonsoo Kim unsigned long entries[4]; 62af52bf6bSThomas Gleixner unsigned int nr_entries; 63f2ca0b55SJoonsoo Kim 64af52bf6bSThomas Gleixner nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 0); 65af52bf6bSThomas Gleixner return stack_depot_save(entries, nr_entries, GFP_KERNEL); 66dab4ead1SVlastimil Babka } 67dab4ead1SVlastimil Babka 68dab4ead1SVlastimil Babka static noinline void register_dummy_stack(void) 69dab4ead1SVlastimil Babka { 70dab4ead1SVlastimil Babka dummy_handle = create_dummy_stack(); 71f2ca0b55SJoonsoo Kim } 72f2ca0b55SJoonsoo Kim 73f2ca0b55SJoonsoo Kim static noinline void register_failure_stack(void) 74f2ca0b55SJoonsoo Kim { 75dab4ead1SVlastimil Babka failure_handle = create_dummy_stack(); 76dab4ead1SVlastimil Babka } 77f2ca0b55SJoonsoo Kim 78dab4ead1SVlastimil Babka static noinline void register_early_stack(void) 79dab4ead1SVlastimil Babka { 80dab4ead1SVlastimil Babka early_handle = create_dummy_stack(); 81f2ca0b55SJoonsoo Kim } 82f2ca0b55SJoonsoo Kim 8348c96a36SJoonsoo Kim static void init_page_owner(void) 8448c96a36SJoonsoo Kim { 850fe9a448SVlastimil Babka if (!page_owner_enabled) 8648c96a36SJoonsoo Kim return; 8748c96a36SJoonsoo Kim 88f2ca0b55SJoonsoo Kim register_dummy_stack(); 89f2ca0b55SJoonsoo Kim register_failure_stack(); 90dab4ead1SVlastimil Babka register_early_stack(); 917dd80b8aSVlastimil Babka static_branch_enable(&page_owner_inited); 9261cf5febSJoonsoo Kim init_early_allocated_pages(); 9348c96a36SJoonsoo Kim } 9448c96a36SJoonsoo Kim 9548c96a36SJoonsoo Kim struct page_ext_operations page_owner_ops = { 969300d8dfSJoonsoo Kim .size = sizeof(struct page_owner), 9748c96a36SJoonsoo Kim .need = need_page_owner, 9848c96a36SJoonsoo Kim .init = init_page_owner, 9948c96a36SJoonsoo Kim }; 10048c96a36SJoonsoo Kim 1019300d8dfSJoonsoo Kim static inline struct page_owner *get_page_owner(struct page_ext *page_ext) 1029300d8dfSJoonsoo Kim { 1039300d8dfSJoonsoo Kim return (void *)page_ext + page_owner_ops.offset; 1049300d8dfSJoonsoo Kim } 1059300d8dfSJoonsoo Kim 106af52bf6bSThomas Gleixner static inline bool check_recursive_alloc(unsigned long *entries, 107af52bf6bSThomas Gleixner unsigned int nr_entries, 108f2ca0b55SJoonsoo Kim unsigned long ip) 10948c96a36SJoonsoo Kim { 110af52bf6bSThomas Gleixner unsigned int i; 111f86e4271SYang Shi 112af52bf6bSThomas Gleixner for (i = 0; i < nr_entries; i++) { 113af52bf6bSThomas Gleixner if (entries[i] == ip) 114f2ca0b55SJoonsoo Kim return true; 115f2ca0b55SJoonsoo Kim } 116f2ca0b55SJoonsoo Kim return false; 117f2ca0b55SJoonsoo Kim } 118f2ca0b55SJoonsoo Kim 119f2ca0b55SJoonsoo Kim static noinline depot_stack_handle_t save_stack(gfp_t flags) 120f2ca0b55SJoonsoo Kim { 121f2ca0b55SJoonsoo Kim unsigned long entries[PAGE_OWNER_STACK_DEPTH]; 122f2ca0b55SJoonsoo Kim depot_stack_handle_t handle; 123af52bf6bSThomas Gleixner unsigned int nr_entries; 124f2ca0b55SJoonsoo Kim 125af52bf6bSThomas Gleixner nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 2); 126f2ca0b55SJoonsoo Kim 127f2ca0b55SJoonsoo Kim /* 128af52bf6bSThomas Gleixner * We need to check recursion here because our request to 129af52bf6bSThomas Gleixner * stackdepot could trigger memory allocation to save new 130af52bf6bSThomas Gleixner * entry. New memory allocation would reach here and call 131af52bf6bSThomas Gleixner * stack_depot_save_entries() again if we don't catch it. There is 132af52bf6bSThomas Gleixner * still not enough memory in stackdepot so it would try to 133af52bf6bSThomas Gleixner * allocate memory again and loop forever. 134f2ca0b55SJoonsoo Kim */ 135af52bf6bSThomas Gleixner if (check_recursive_alloc(entries, nr_entries, _RET_IP_)) 136f2ca0b55SJoonsoo Kim return dummy_handle; 137f2ca0b55SJoonsoo Kim 138af52bf6bSThomas Gleixner handle = stack_depot_save(entries, nr_entries, flags); 139f2ca0b55SJoonsoo Kim if (!handle) 140f2ca0b55SJoonsoo Kim handle = failure_handle; 141f2ca0b55SJoonsoo Kim 142f2ca0b55SJoonsoo Kim return handle; 143f2ca0b55SJoonsoo Kim } 144f2ca0b55SJoonsoo Kim 1458974558fSVlastimil Babka void __reset_page_owner(struct page *page, unsigned int order) 1468974558fSVlastimil Babka { 1478974558fSVlastimil Babka int i; 1488974558fSVlastimil Babka struct page_ext *page_ext; 1498974558fSVlastimil Babka depot_stack_handle_t handle = 0; 1508974558fSVlastimil Babka struct page_owner *page_owner; 1518974558fSVlastimil Babka 1528974558fSVlastimil Babka handle = save_stack(GFP_NOWAIT | __GFP_NOWARN); 1538974558fSVlastimil Babka 1545556cfe8SVlastimil Babka page_ext = lookup_page_ext(page); 1558974558fSVlastimil Babka if (unlikely(!page_ext)) 1565556cfe8SVlastimil Babka return; 1575556cfe8SVlastimil Babka for (i = 0; i < (1 << order); i++) { 158fdf3bf80SVlastimil Babka __clear_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags); 1598974558fSVlastimil Babka page_owner = get_page_owner(page_ext); 1608974558fSVlastimil Babka page_owner->free_handle = handle; 1615556cfe8SVlastimil Babka page_ext = page_ext_next(page_ext); 1628974558fSVlastimil Babka } 1638974558fSVlastimil Babka } 1648974558fSVlastimil Babka 1657e2f2a0cSVlastimil Babka static inline void __set_page_owner_handle(struct page *page, 1667e2f2a0cSVlastimil Babka struct page_ext *page_ext, depot_stack_handle_t handle, 1677e2f2a0cSVlastimil Babka unsigned int order, gfp_t gfp_mask) 168f2ca0b55SJoonsoo Kim { 1699300d8dfSJoonsoo Kim struct page_owner *page_owner; 1707e2f2a0cSVlastimil Babka int i; 17148c96a36SJoonsoo Kim 1727e2f2a0cSVlastimil Babka for (i = 0; i < (1 << order); i++) { 1739300d8dfSJoonsoo Kim page_owner = get_page_owner(page_ext); 174dab4ead1SVlastimil Babka page_owner->handle = handle; 1759300d8dfSJoonsoo Kim page_owner->order = order; 1769300d8dfSJoonsoo Kim page_owner->gfp_mask = gfp_mask; 1779300d8dfSJoonsoo Kim page_owner->last_migrate_reason = -1; 1789cc7e96aSLiam Mark page_owner->pid = current->pid; 1799cc7e96aSLiam Mark page_owner->ts_nsec = local_clock(); 18048c96a36SJoonsoo Kim __set_bit(PAGE_EXT_OWNER, &page_ext->flags); 181fdf3bf80SVlastimil Babka __set_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags); 1827e2f2a0cSVlastimil Babka 1835556cfe8SVlastimil Babka page_ext = page_ext_next(page_ext); 1847e2f2a0cSVlastimil Babka } 18548c96a36SJoonsoo Kim } 18648c96a36SJoonsoo Kim 187dab4ead1SVlastimil Babka noinline void __set_page_owner(struct page *page, unsigned int order, 188dab4ead1SVlastimil Babka gfp_t gfp_mask) 189dab4ead1SVlastimil Babka { 190dab4ead1SVlastimil Babka struct page_ext *page_ext = lookup_page_ext(page); 191dab4ead1SVlastimil Babka depot_stack_handle_t handle; 192dab4ead1SVlastimil Babka 193dab4ead1SVlastimil Babka if (unlikely(!page_ext)) 194dab4ead1SVlastimil Babka return; 195dab4ead1SVlastimil Babka 196dab4ead1SVlastimil Babka handle = save_stack(gfp_mask); 1977e2f2a0cSVlastimil Babka __set_page_owner_handle(page, page_ext, handle, order, gfp_mask); 198dab4ead1SVlastimil Babka } 199dab4ead1SVlastimil Babka 2007cd12b4aSVlastimil Babka void __set_page_owner_migrate_reason(struct page *page, int reason) 2017cd12b4aSVlastimil Babka { 2027cd12b4aSVlastimil Babka struct page_ext *page_ext = lookup_page_ext(page); 2039300d8dfSJoonsoo Kim struct page_owner *page_owner; 2049300d8dfSJoonsoo Kim 205f86e4271SYang Shi if (unlikely(!page_ext)) 206f86e4271SYang Shi return; 2077cd12b4aSVlastimil Babka 2089300d8dfSJoonsoo Kim page_owner = get_page_owner(page_ext); 2099300d8dfSJoonsoo Kim page_owner->last_migrate_reason = reason; 2107cd12b4aSVlastimil Babka } 2117cd12b4aSVlastimil Babka 2128fb156c9SMatthew Wilcox (Oracle) void __split_page_owner(struct page *page, unsigned int nr) 213e2cfc911SJoonsoo Kim { 214a9627bc5SJoonsoo Kim int i; 215e2cfc911SJoonsoo Kim struct page_ext *page_ext = lookup_page_ext(page); 2169300d8dfSJoonsoo Kim struct page_owner *page_owner; 217e2cfc911SJoonsoo Kim 218a9627bc5SJoonsoo Kim if (unlikely(!page_ext)) 219a9627bc5SJoonsoo Kim return; 220a9627bc5SJoonsoo Kim 2218fb156c9SMatthew Wilcox (Oracle) for (i = 0; i < nr; i++) { 2229300d8dfSJoonsoo Kim page_owner = get_page_owner(page_ext); 2239300d8dfSJoonsoo Kim page_owner->order = 0; 2245556cfe8SVlastimil Babka page_ext = page_ext_next(page_ext); 2257e2f2a0cSVlastimil Babka } 226e2cfc911SJoonsoo Kim } 227e2cfc911SJoonsoo Kim 228d435edcaSVlastimil Babka void __copy_page_owner(struct page *oldpage, struct page *newpage) 229d435edcaSVlastimil Babka { 230d435edcaSVlastimil Babka struct page_ext *old_ext = lookup_page_ext(oldpage); 231d435edcaSVlastimil Babka struct page_ext *new_ext = lookup_page_ext(newpage); 2329300d8dfSJoonsoo Kim struct page_owner *old_page_owner, *new_page_owner; 233d435edcaSVlastimil Babka 234f86e4271SYang Shi if (unlikely(!old_ext || !new_ext)) 235f86e4271SYang Shi return; 236f86e4271SYang Shi 2379300d8dfSJoonsoo Kim old_page_owner = get_page_owner(old_ext); 2389300d8dfSJoonsoo Kim new_page_owner = get_page_owner(new_ext); 2399300d8dfSJoonsoo Kim new_page_owner->order = old_page_owner->order; 2409300d8dfSJoonsoo Kim new_page_owner->gfp_mask = old_page_owner->gfp_mask; 2419300d8dfSJoonsoo Kim new_page_owner->last_migrate_reason = 2429300d8dfSJoonsoo Kim old_page_owner->last_migrate_reason; 2439300d8dfSJoonsoo Kim new_page_owner->handle = old_page_owner->handle; 2449cc7e96aSLiam Mark new_page_owner->pid = old_page_owner->pid; 2459cc7e96aSLiam Mark new_page_owner->ts_nsec = old_page_owner->ts_nsec; 246d435edcaSVlastimil Babka 247d435edcaSVlastimil Babka /* 248d435edcaSVlastimil Babka * We don't clear the bit on the oldpage as it's going to be freed 249d435edcaSVlastimil Babka * after migration. Until then, the info can be useful in case of 250d435edcaSVlastimil Babka * a bug, and the overal stats will be off a bit only temporarily. 251d435edcaSVlastimil Babka * Also, migrate_misplaced_transhuge_page() can still fail the 252d435edcaSVlastimil Babka * migration and then we want the oldpage to retain the info. But 253d435edcaSVlastimil Babka * in that case we also don't need to explicitly clear the info from 254d435edcaSVlastimil Babka * the new page, which will be freed. 255d435edcaSVlastimil Babka */ 256d435edcaSVlastimil Babka __set_bit(PAGE_EXT_OWNER, &new_ext->flags); 257fdf3bf80SVlastimil Babka __set_bit(PAGE_EXT_OWNER_ALLOCATED, &new_ext->flags); 258d435edcaSVlastimil Babka } 259d435edcaSVlastimil Babka 260e2f612e6SJoonsoo Kim void pagetypeinfo_showmixedcount_print(struct seq_file *m, 261e2f612e6SJoonsoo Kim pg_data_t *pgdat, struct zone *zone) 262e2f612e6SJoonsoo Kim { 263e2f612e6SJoonsoo Kim struct page *page; 264e2f612e6SJoonsoo Kim struct page_ext *page_ext; 2659300d8dfSJoonsoo Kim struct page_owner *page_owner; 266*1d2cae8eSMiaohe Lin unsigned long pfn, block_end_pfn; 267*1d2cae8eSMiaohe Lin unsigned long end_pfn = zone_end_pfn(zone); 268e2f612e6SJoonsoo Kim unsigned long count[MIGRATE_TYPES] = { 0, }; 269e2f612e6SJoonsoo Kim int pageblock_mt, page_mt; 270e2f612e6SJoonsoo Kim int i; 271e2f612e6SJoonsoo Kim 272e2f612e6SJoonsoo Kim /* Scan block by block. First and last block may be incomplete */ 273e2f612e6SJoonsoo Kim pfn = zone->zone_start_pfn; 274e2f612e6SJoonsoo Kim 275e2f612e6SJoonsoo Kim /* 276e2f612e6SJoonsoo Kim * Walk the zone in pageblock_nr_pages steps. If a page block spans 277e2f612e6SJoonsoo Kim * a zone boundary, it will be double counted between zones. This does 278e2f612e6SJoonsoo Kim * not matter as the mixed block count will still be correct 279e2f612e6SJoonsoo Kim */ 280e2f612e6SJoonsoo Kim for (; pfn < end_pfn; ) { 281a26ee565SQian Cai page = pfn_to_online_page(pfn); 282a26ee565SQian Cai if (!page) { 283e2f612e6SJoonsoo Kim pfn = ALIGN(pfn + 1, MAX_ORDER_NR_PAGES); 284e2f612e6SJoonsoo Kim continue; 285e2f612e6SJoonsoo Kim } 286e2f612e6SJoonsoo Kim 287e2f612e6SJoonsoo Kim block_end_pfn = ALIGN(pfn + 1, pageblock_nr_pages); 288e2f612e6SJoonsoo Kim block_end_pfn = min(block_end_pfn, end_pfn); 289e2f612e6SJoonsoo Kim 290e2f612e6SJoonsoo Kim pageblock_mt = get_pageblock_migratetype(page); 291e2f612e6SJoonsoo Kim 292e2f612e6SJoonsoo Kim for (; pfn < block_end_pfn; pfn++) { 293e2f612e6SJoonsoo Kim if (!pfn_valid_within(pfn)) 294e2f612e6SJoonsoo Kim continue; 295e2f612e6SJoonsoo Kim 296a26ee565SQian Cai /* The pageblock is online, no need to recheck. */ 297e2f612e6SJoonsoo Kim page = pfn_to_page(pfn); 298e2f612e6SJoonsoo Kim 299e2f612e6SJoonsoo Kim if (page_zone(page) != zone) 300e2f612e6SJoonsoo Kim continue; 301e2f612e6SJoonsoo Kim 302e2f612e6SJoonsoo Kim if (PageBuddy(page)) { 303727c080fSVinayak Menon unsigned long freepage_order; 304727c080fSVinayak Menon 305ab130f91SMatthew Wilcox (Oracle) freepage_order = buddy_order_unsafe(page); 306727c080fSVinayak Menon if (freepage_order < MAX_ORDER) 307727c080fSVinayak Menon pfn += (1UL << freepage_order) - 1; 308e2f612e6SJoonsoo Kim continue; 309e2f612e6SJoonsoo Kim } 310e2f612e6SJoonsoo Kim 311e2f612e6SJoonsoo Kim if (PageReserved(page)) 312e2f612e6SJoonsoo Kim continue; 313e2f612e6SJoonsoo Kim 314e2f612e6SJoonsoo Kim page_ext = lookup_page_ext(page); 315e2f612e6SJoonsoo Kim if (unlikely(!page_ext)) 316e2f612e6SJoonsoo Kim continue; 317e2f612e6SJoonsoo Kim 318fdf3bf80SVlastimil Babka if (!test_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags)) 319e2f612e6SJoonsoo Kim continue; 320e2f612e6SJoonsoo Kim 3219300d8dfSJoonsoo Kim page_owner = get_page_owner(page_ext); 32201c0bfe0SWei Yang page_mt = gfp_migratetype(page_owner->gfp_mask); 323e2f612e6SJoonsoo Kim if (pageblock_mt != page_mt) { 324e2f612e6SJoonsoo Kim if (is_migrate_cma(pageblock_mt)) 325e2f612e6SJoonsoo Kim count[MIGRATE_MOVABLE]++; 326e2f612e6SJoonsoo Kim else 327e2f612e6SJoonsoo Kim count[pageblock_mt]++; 328e2f612e6SJoonsoo Kim 329e2f612e6SJoonsoo Kim pfn = block_end_pfn; 330e2f612e6SJoonsoo Kim break; 331e2f612e6SJoonsoo Kim } 3329300d8dfSJoonsoo Kim pfn += (1UL << page_owner->order) - 1; 333e2f612e6SJoonsoo Kim } 334e2f612e6SJoonsoo Kim } 335e2f612e6SJoonsoo Kim 336e2f612e6SJoonsoo Kim /* Print counts */ 337e2f612e6SJoonsoo Kim seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name); 338e2f612e6SJoonsoo Kim for (i = 0; i < MIGRATE_TYPES; i++) 339e2f612e6SJoonsoo Kim seq_printf(m, "%12lu ", count[i]); 340e2f612e6SJoonsoo Kim seq_putc(m, '\n'); 341e2f612e6SJoonsoo Kim } 342e2f612e6SJoonsoo Kim 34348c96a36SJoonsoo Kim static ssize_t 34448c96a36SJoonsoo Kim print_page_owner(char __user *buf, size_t count, unsigned long pfn, 3459300d8dfSJoonsoo Kim struct page *page, struct page_owner *page_owner, 346f2ca0b55SJoonsoo Kim depot_stack_handle_t handle) 34748c96a36SJoonsoo Kim { 348af52bf6bSThomas Gleixner int ret, pageblock_mt, page_mt; 349af52bf6bSThomas Gleixner unsigned long *entries; 350af52bf6bSThomas Gleixner unsigned int nr_entries; 35148c96a36SJoonsoo Kim char *kbuf; 35248c96a36SJoonsoo Kim 353c8f61cfcSMiles Chen count = min_t(size_t, count, PAGE_SIZE); 35448c96a36SJoonsoo Kim kbuf = kmalloc(count, GFP_KERNEL); 35548c96a36SJoonsoo Kim if (!kbuf) 35648c96a36SJoonsoo Kim return -ENOMEM; 35748c96a36SJoonsoo Kim 35848c96a36SJoonsoo Kim ret = snprintf(kbuf, count, 3599cc7e96aSLiam Mark "Page allocated via order %u, mask %#x(%pGg), pid %d, ts %llu ns\n", 3609300d8dfSJoonsoo Kim page_owner->order, page_owner->gfp_mask, 3619cc7e96aSLiam Mark &page_owner->gfp_mask, page_owner->pid, 3629cc7e96aSLiam Mark page_owner->ts_nsec); 36348c96a36SJoonsoo Kim 36448c96a36SJoonsoo Kim if (ret >= count) 36548c96a36SJoonsoo Kim goto err; 36648c96a36SJoonsoo Kim 36748c96a36SJoonsoo Kim /* Print information relevant to grouping pages by mobility */ 3680b423ca2SMel Gorman pageblock_mt = get_pageblock_migratetype(page); 36901c0bfe0SWei Yang page_mt = gfp_migratetype(page_owner->gfp_mask); 37048c96a36SJoonsoo Kim ret += snprintf(kbuf + ret, count - ret, 37160f30350SVlastimil Babka "PFN %lu type %s Block %lu type %s Flags %#lx(%pGp)\n", 37248c96a36SJoonsoo Kim pfn, 37360f30350SVlastimil Babka migratetype_names[page_mt], 37448c96a36SJoonsoo Kim pfn >> pageblock_order, 37560f30350SVlastimil Babka migratetype_names[pageblock_mt], 37660f30350SVlastimil Babka page->flags, &page->flags); 37748c96a36SJoonsoo Kim 37848c96a36SJoonsoo Kim if (ret >= count) 37948c96a36SJoonsoo Kim goto err; 38048c96a36SJoonsoo Kim 381af52bf6bSThomas Gleixner nr_entries = stack_depot_fetch(handle, &entries); 382af52bf6bSThomas Gleixner ret += stack_trace_snprint(kbuf + ret, count - ret, entries, nr_entries, 0); 38348c96a36SJoonsoo Kim if (ret >= count) 38448c96a36SJoonsoo Kim goto err; 38548c96a36SJoonsoo Kim 3869300d8dfSJoonsoo Kim if (page_owner->last_migrate_reason != -1) { 3877cd12b4aSVlastimil Babka ret += snprintf(kbuf + ret, count - ret, 3887cd12b4aSVlastimil Babka "Page has been migrated, last migrate reason: %s\n", 3899300d8dfSJoonsoo Kim migrate_reason_names[page_owner->last_migrate_reason]); 3907cd12b4aSVlastimil Babka if (ret >= count) 3917cd12b4aSVlastimil Babka goto err; 3927cd12b4aSVlastimil Babka } 3937cd12b4aSVlastimil Babka 39448c96a36SJoonsoo Kim ret += snprintf(kbuf + ret, count - ret, "\n"); 39548c96a36SJoonsoo Kim if (ret >= count) 39648c96a36SJoonsoo Kim goto err; 39748c96a36SJoonsoo Kim 39848c96a36SJoonsoo Kim if (copy_to_user(buf, kbuf, ret)) 39948c96a36SJoonsoo Kim ret = -EFAULT; 40048c96a36SJoonsoo Kim 40148c96a36SJoonsoo Kim kfree(kbuf); 40248c96a36SJoonsoo Kim return ret; 40348c96a36SJoonsoo Kim 40448c96a36SJoonsoo Kim err: 40548c96a36SJoonsoo Kim kfree(kbuf); 40648c96a36SJoonsoo Kim return -ENOMEM; 40748c96a36SJoonsoo Kim } 40848c96a36SJoonsoo Kim 4094e462112SVlastimil Babka void __dump_page_owner(struct page *page) 4104e462112SVlastimil Babka { 4114e462112SVlastimil Babka struct page_ext *page_ext = lookup_page_ext(page); 4129300d8dfSJoonsoo Kim struct page_owner *page_owner; 413f2ca0b55SJoonsoo Kim depot_stack_handle_t handle; 414af52bf6bSThomas Gleixner unsigned long *entries; 415af52bf6bSThomas Gleixner unsigned int nr_entries; 4168285027fSSudip Mukherjee gfp_t gfp_mask; 4178285027fSSudip Mukherjee int mt; 4184e462112SVlastimil Babka 419f86e4271SYang Shi if (unlikely(!page_ext)) { 420f86e4271SYang Shi pr_alert("There is not page extension available.\n"); 421f86e4271SYang Shi return; 422f86e4271SYang Shi } 4239300d8dfSJoonsoo Kim 4249300d8dfSJoonsoo Kim page_owner = get_page_owner(page_ext); 4259300d8dfSJoonsoo Kim gfp_mask = page_owner->gfp_mask; 42601c0bfe0SWei Yang mt = gfp_migratetype(gfp_mask); 427f86e4271SYang Shi 4284e462112SVlastimil Babka if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags)) { 42937389167SVlastimil Babka pr_alert("page_owner info is not present (never set?)\n"); 4304e462112SVlastimil Babka return; 4314e462112SVlastimil Babka } 4324e462112SVlastimil Babka 433fdf3bf80SVlastimil Babka if (test_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags)) 43437389167SVlastimil Babka pr_alert("page_owner tracks the page as allocated\n"); 43537389167SVlastimil Babka else 43637389167SVlastimil Babka pr_alert("page_owner tracks the page as freed\n"); 43737389167SVlastimil Babka 4389cc7e96aSLiam Mark pr_alert("page last allocated via order %u, migratetype %s, gfp_mask %#x(%pGg), pid %d, ts %llu\n", 4399cc7e96aSLiam Mark page_owner->order, migratetype_names[mt], gfp_mask, &gfp_mask, 4409cc7e96aSLiam Mark page_owner->pid, page_owner->ts_nsec); 44137389167SVlastimil Babka 4429300d8dfSJoonsoo Kim handle = READ_ONCE(page_owner->handle); 443f2ca0b55SJoonsoo Kim if (!handle) { 44437389167SVlastimil Babka pr_alert("page_owner allocation stack trace missing\n"); 44537389167SVlastimil Babka } else { 446af52bf6bSThomas Gleixner nr_entries = stack_depot_fetch(handle, &entries); 447af52bf6bSThomas Gleixner stack_trace_print(entries, nr_entries, 0); 44837389167SVlastimil Babka } 4494e462112SVlastimil Babka 4508974558fSVlastimil Babka handle = READ_ONCE(page_owner->free_handle); 4518974558fSVlastimil Babka if (!handle) { 4528974558fSVlastimil Babka pr_alert("page_owner free stack trace missing\n"); 4538974558fSVlastimil Babka } else { 4548974558fSVlastimil Babka nr_entries = stack_depot_fetch(handle, &entries); 4558974558fSVlastimil Babka pr_alert("page last free stack trace:\n"); 4568974558fSVlastimil Babka stack_trace_print(entries, nr_entries, 0); 4578974558fSVlastimil Babka } 4588974558fSVlastimil Babka 4599300d8dfSJoonsoo Kim if (page_owner->last_migrate_reason != -1) 4604e462112SVlastimil Babka pr_alert("page has been migrated, last migrate reason: %s\n", 4619300d8dfSJoonsoo Kim migrate_reason_names[page_owner->last_migrate_reason]); 4624e462112SVlastimil Babka } 4634e462112SVlastimil Babka 46448c96a36SJoonsoo Kim static ssize_t 46548c96a36SJoonsoo Kim read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos) 46648c96a36SJoonsoo Kim { 46748c96a36SJoonsoo Kim unsigned long pfn; 46848c96a36SJoonsoo Kim struct page *page; 46948c96a36SJoonsoo Kim struct page_ext *page_ext; 4709300d8dfSJoonsoo Kim struct page_owner *page_owner; 471f2ca0b55SJoonsoo Kim depot_stack_handle_t handle; 47248c96a36SJoonsoo Kim 4737dd80b8aSVlastimil Babka if (!static_branch_unlikely(&page_owner_inited)) 47448c96a36SJoonsoo Kim return -EINVAL; 47548c96a36SJoonsoo Kim 47648c96a36SJoonsoo Kim page = NULL; 47748c96a36SJoonsoo Kim pfn = min_low_pfn + *ppos; 47848c96a36SJoonsoo Kim 47948c96a36SJoonsoo Kim /* Find a valid PFN or the start of a MAX_ORDER_NR_PAGES area */ 48048c96a36SJoonsoo Kim while (!pfn_valid(pfn) && (pfn & (MAX_ORDER_NR_PAGES - 1)) != 0) 48148c96a36SJoonsoo Kim pfn++; 48248c96a36SJoonsoo Kim 48348c96a36SJoonsoo Kim drain_all_pages(NULL); 48448c96a36SJoonsoo Kim 48548c96a36SJoonsoo Kim /* Find an allocated page */ 48648c96a36SJoonsoo Kim for (; pfn < max_pfn; pfn++) { 48748c96a36SJoonsoo Kim /* 48848c96a36SJoonsoo Kim * If the new page is in a new MAX_ORDER_NR_PAGES area, 48948c96a36SJoonsoo Kim * validate the area as existing, skip it if not 49048c96a36SJoonsoo Kim */ 49148c96a36SJoonsoo Kim if ((pfn & (MAX_ORDER_NR_PAGES - 1)) == 0 && !pfn_valid(pfn)) { 49248c96a36SJoonsoo Kim pfn += MAX_ORDER_NR_PAGES - 1; 49348c96a36SJoonsoo Kim continue; 49448c96a36SJoonsoo Kim } 49548c96a36SJoonsoo Kim 49648c96a36SJoonsoo Kim /* Check for holes within a MAX_ORDER area */ 49748c96a36SJoonsoo Kim if (!pfn_valid_within(pfn)) 49848c96a36SJoonsoo Kim continue; 49948c96a36SJoonsoo Kim 50048c96a36SJoonsoo Kim page = pfn_to_page(pfn); 50148c96a36SJoonsoo Kim if (PageBuddy(page)) { 502ab130f91SMatthew Wilcox (Oracle) unsigned long freepage_order = buddy_order_unsafe(page); 50348c96a36SJoonsoo Kim 50448c96a36SJoonsoo Kim if (freepage_order < MAX_ORDER) 50548c96a36SJoonsoo Kim pfn += (1UL << freepage_order) - 1; 50648c96a36SJoonsoo Kim continue; 50748c96a36SJoonsoo Kim } 50848c96a36SJoonsoo Kim 50948c96a36SJoonsoo Kim page_ext = lookup_page_ext(page); 510f86e4271SYang Shi if (unlikely(!page_ext)) 511f86e4271SYang Shi continue; 51248c96a36SJoonsoo Kim 51348c96a36SJoonsoo Kim /* 51461cf5febSJoonsoo Kim * Some pages could be missed by concurrent allocation or free, 51561cf5febSJoonsoo Kim * because we don't hold the zone lock. 51648c96a36SJoonsoo Kim */ 51748c96a36SJoonsoo Kim if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags)) 51848c96a36SJoonsoo Kim continue; 51948c96a36SJoonsoo Kim 52037389167SVlastimil Babka /* 52137389167SVlastimil Babka * Although we do have the info about past allocation of free 52237389167SVlastimil Babka * pages, it's not relevant for current memory usage. 52337389167SVlastimil Babka */ 524fdf3bf80SVlastimil Babka if (!test_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags)) 52537389167SVlastimil Babka continue; 52637389167SVlastimil Babka 5279300d8dfSJoonsoo Kim page_owner = get_page_owner(page_ext); 5289300d8dfSJoonsoo Kim 529f2ca0b55SJoonsoo Kim /* 5307e2f2a0cSVlastimil Babka * Don't print "tail" pages of high-order allocations as that 5317e2f2a0cSVlastimil Babka * would inflate the stats. 5327e2f2a0cSVlastimil Babka */ 5337e2f2a0cSVlastimil Babka if (!IS_ALIGNED(pfn, 1 << page_owner->order)) 5347e2f2a0cSVlastimil Babka continue; 5357e2f2a0cSVlastimil Babka 5367e2f2a0cSVlastimil Babka /* 537f2ca0b55SJoonsoo Kim * Access to page_ext->handle isn't synchronous so we should 538f2ca0b55SJoonsoo Kim * be careful to access it. 539f2ca0b55SJoonsoo Kim */ 5409300d8dfSJoonsoo Kim handle = READ_ONCE(page_owner->handle); 541f2ca0b55SJoonsoo Kim if (!handle) 542f2ca0b55SJoonsoo Kim continue; 543f2ca0b55SJoonsoo Kim 54448c96a36SJoonsoo Kim /* Record the next PFN to read in the file offset */ 54548c96a36SJoonsoo Kim *ppos = (pfn - min_low_pfn) + 1; 54648c96a36SJoonsoo Kim 547f2ca0b55SJoonsoo Kim return print_page_owner(buf, count, pfn, page, 5489300d8dfSJoonsoo Kim page_owner, handle); 54948c96a36SJoonsoo Kim } 55048c96a36SJoonsoo Kim 55148c96a36SJoonsoo Kim return 0; 55248c96a36SJoonsoo Kim } 55348c96a36SJoonsoo Kim 55461cf5febSJoonsoo Kim static void init_pages_in_zone(pg_data_t *pgdat, struct zone *zone) 55561cf5febSJoonsoo Kim { 5566787c1daSOscar Salvador unsigned long pfn = zone->zone_start_pfn; 5576787c1daSOscar Salvador unsigned long end_pfn = zone_end_pfn(zone); 55861cf5febSJoonsoo Kim unsigned long count = 0; 55961cf5febSJoonsoo Kim 56061cf5febSJoonsoo Kim /* 56161cf5febSJoonsoo Kim * Walk the zone in pageblock_nr_pages steps. If a page block spans 56261cf5febSJoonsoo Kim * a zone boundary, it will be double counted between zones. This does 56361cf5febSJoonsoo Kim * not matter as the mixed block count will still be correct 56461cf5febSJoonsoo Kim */ 56561cf5febSJoonsoo Kim for (; pfn < end_pfn; ) { 5666787c1daSOscar Salvador unsigned long block_end_pfn; 5676787c1daSOscar Salvador 56861cf5febSJoonsoo Kim if (!pfn_valid(pfn)) { 56961cf5febSJoonsoo Kim pfn = ALIGN(pfn + 1, MAX_ORDER_NR_PAGES); 57061cf5febSJoonsoo Kim continue; 57161cf5febSJoonsoo Kim } 57261cf5febSJoonsoo Kim 57361cf5febSJoonsoo Kim block_end_pfn = ALIGN(pfn + 1, pageblock_nr_pages); 57461cf5febSJoonsoo Kim block_end_pfn = min(block_end_pfn, end_pfn); 57561cf5febSJoonsoo Kim 57661cf5febSJoonsoo Kim for (; pfn < block_end_pfn; pfn++) { 5776787c1daSOscar Salvador struct page *page; 5786787c1daSOscar Salvador struct page_ext *page_ext; 5796787c1daSOscar Salvador 58061cf5febSJoonsoo Kim if (!pfn_valid_within(pfn)) 58161cf5febSJoonsoo Kim continue; 58261cf5febSJoonsoo Kim 58361cf5febSJoonsoo Kim page = pfn_to_page(pfn); 58461cf5febSJoonsoo Kim 5859d43f5aeSJoonsoo Kim if (page_zone(page) != zone) 5869d43f5aeSJoonsoo Kim continue; 5879d43f5aeSJoonsoo Kim 58861cf5febSJoonsoo Kim /* 58910903027SVlastimil Babka * To avoid having to grab zone->lock, be a little 59010903027SVlastimil Babka * careful when reading buddy page order. The only 59110903027SVlastimil Babka * danger is that we skip too much and potentially miss 59210903027SVlastimil Babka * some early allocated pages, which is better than 59310903027SVlastimil Babka * heavy lock contention. 59461cf5febSJoonsoo Kim */ 59561cf5febSJoonsoo Kim if (PageBuddy(page)) { 596ab130f91SMatthew Wilcox (Oracle) unsigned long order = buddy_order_unsafe(page); 59710903027SVlastimil Babka 59810903027SVlastimil Babka if (order > 0 && order < MAX_ORDER) 59910903027SVlastimil Babka pfn += (1UL << order) - 1; 60061cf5febSJoonsoo Kim continue; 60161cf5febSJoonsoo Kim } 60261cf5febSJoonsoo Kim 60361cf5febSJoonsoo Kim if (PageReserved(page)) 60461cf5febSJoonsoo Kim continue; 60561cf5febSJoonsoo Kim 60661cf5febSJoonsoo Kim page_ext = lookup_page_ext(page); 607f86e4271SYang Shi if (unlikely(!page_ext)) 608f86e4271SYang Shi continue; 60961cf5febSJoonsoo Kim 610dab4ead1SVlastimil Babka /* Maybe overlapping zone */ 61161cf5febSJoonsoo Kim if (test_bit(PAGE_EXT_OWNER, &page_ext->flags)) 61261cf5febSJoonsoo Kim continue; 61361cf5febSJoonsoo Kim 61461cf5febSJoonsoo Kim /* Found early allocated page */ 6157e2f2a0cSVlastimil Babka __set_page_owner_handle(page, page_ext, early_handle, 6167e2f2a0cSVlastimil Babka 0, 0); 61761cf5febSJoonsoo Kim count++; 61861cf5febSJoonsoo Kim } 61910903027SVlastimil Babka cond_resched(); 62061cf5febSJoonsoo Kim } 62161cf5febSJoonsoo Kim 62261cf5febSJoonsoo Kim pr_info("Node %d, zone %8s: page owner found early allocated %lu pages\n", 62361cf5febSJoonsoo Kim pgdat->node_id, zone->name, count); 62461cf5febSJoonsoo Kim } 62561cf5febSJoonsoo Kim 62661cf5febSJoonsoo Kim static void init_zones_in_node(pg_data_t *pgdat) 62761cf5febSJoonsoo Kim { 62861cf5febSJoonsoo Kim struct zone *zone; 62961cf5febSJoonsoo Kim struct zone *node_zones = pgdat->node_zones; 63061cf5febSJoonsoo Kim 63161cf5febSJoonsoo Kim for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) { 63261cf5febSJoonsoo Kim if (!populated_zone(zone)) 63361cf5febSJoonsoo Kim continue; 63461cf5febSJoonsoo Kim 63561cf5febSJoonsoo Kim init_pages_in_zone(pgdat, zone); 63661cf5febSJoonsoo Kim } 63761cf5febSJoonsoo Kim } 63861cf5febSJoonsoo Kim 63961cf5febSJoonsoo Kim static void init_early_allocated_pages(void) 64061cf5febSJoonsoo Kim { 64161cf5febSJoonsoo Kim pg_data_t *pgdat; 64261cf5febSJoonsoo Kim 64361cf5febSJoonsoo Kim for_each_online_pgdat(pgdat) 64461cf5febSJoonsoo Kim init_zones_in_node(pgdat); 64561cf5febSJoonsoo Kim } 64661cf5febSJoonsoo Kim 64748c96a36SJoonsoo Kim static const struct file_operations proc_page_owner_operations = { 64848c96a36SJoonsoo Kim .read = read_page_owner, 64948c96a36SJoonsoo Kim }; 65048c96a36SJoonsoo Kim 65148c96a36SJoonsoo Kim static int __init pageowner_init(void) 65248c96a36SJoonsoo Kim { 6537dd80b8aSVlastimil Babka if (!static_branch_unlikely(&page_owner_inited)) { 65448c96a36SJoonsoo Kim pr_info("page_owner is disabled\n"); 65548c96a36SJoonsoo Kim return 0; 65648c96a36SJoonsoo Kim } 65748c96a36SJoonsoo Kim 658d9f7979cSGreg Kroah-Hartman debugfs_create_file("page_owner", 0400, NULL, NULL, 659d9f7979cSGreg Kroah-Hartman &proc_page_owner_operations); 66048c96a36SJoonsoo Kim 661d9f7979cSGreg Kroah-Hartman return 0; 66248c96a36SJoonsoo Kim } 66344c5af96SPaul Gortmaker late_initcall(pageowner_init) 664