1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 248c96a36SJoonsoo Kim #include <linux/debugfs.h> 348c96a36SJoonsoo Kim #include <linux/mm.h> 448c96a36SJoonsoo Kim #include <linux/slab.h> 548c96a36SJoonsoo Kim #include <linux/uaccess.h> 657c8a661SMike Rapoport #include <linux/memblock.h> 748c96a36SJoonsoo Kim #include <linux/stacktrace.h> 848c96a36SJoonsoo Kim #include <linux/page_owner.h> 97dd80b8aSVlastimil Babka #include <linux/jump_label.h> 107cd12b4aSVlastimil Babka #include <linux/migrate.h> 11f2ca0b55SJoonsoo Kim #include <linux/stackdepot.h> 12e2f612e6SJoonsoo Kim #include <linux/seq_file.h> 13f2ca0b55SJoonsoo Kim 1448c96a36SJoonsoo Kim #include "internal.h" 1548c96a36SJoonsoo Kim 16f2ca0b55SJoonsoo Kim /* 17f2ca0b55SJoonsoo Kim * TODO: teach PAGE_OWNER_STACK_DEPTH (__dump_page_owner and save_stack) 18f2ca0b55SJoonsoo Kim * to use off stack temporal storage 19f2ca0b55SJoonsoo Kim */ 20f2ca0b55SJoonsoo Kim #define PAGE_OWNER_STACK_DEPTH (16) 21f2ca0b55SJoonsoo Kim 229300d8dfSJoonsoo Kim struct page_owner { 236b4c54e3SAyush Mittal unsigned short order; 246b4c54e3SAyush Mittal short last_migrate_reason; 259300d8dfSJoonsoo Kim gfp_t gfp_mask; 269300d8dfSJoonsoo Kim depot_stack_handle_t handle; 279300d8dfSJoonsoo Kim }; 289300d8dfSJoonsoo Kim 2948c96a36SJoonsoo Kim static bool page_owner_disabled = true; 307dd80b8aSVlastimil Babka DEFINE_STATIC_KEY_FALSE(page_owner_inited); 3148c96a36SJoonsoo Kim 32f2ca0b55SJoonsoo Kim static depot_stack_handle_t dummy_handle; 33f2ca0b55SJoonsoo Kim static depot_stack_handle_t failure_handle; 34dab4ead1SVlastimil Babka static depot_stack_handle_t early_handle; 35f2ca0b55SJoonsoo Kim 3661cf5febSJoonsoo Kim static void init_early_allocated_pages(void); 3761cf5febSJoonsoo Kim 381173194eSDou Liyang static int __init early_page_owner_param(char *buf) 3948c96a36SJoonsoo Kim { 4048c96a36SJoonsoo Kim if (!buf) 4148c96a36SJoonsoo Kim return -EINVAL; 4248c96a36SJoonsoo Kim 4348c96a36SJoonsoo Kim if (strcmp(buf, "on") == 0) 4448c96a36SJoonsoo Kim page_owner_disabled = false; 4548c96a36SJoonsoo Kim 4648c96a36SJoonsoo Kim return 0; 4748c96a36SJoonsoo Kim } 4848c96a36SJoonsoo Kim early_param("page_owner", early_page_owner_param); 4948c96a36SJoonsoo Kim 5048c96a36SJoonsoo Kim static bool need_page_owner(void) 5148c96a36SJoonsoo Kim { 5248c96a36SJoonsoo Kim if (page_owner_disabled) 5348c96a36SJoonsoo Kim return false; 5448c96a36SJoonsoo Kim 5548c96a36SJoonsoo Kim return true; 5648c96a36SJoonsoo Kim } 5748c96a36SJoonsoo Kim 58dab4ead1SVlastimil Babka static __always_inline depot_stack_handle_t create_dummy_stack(void) 59f2ca0b55SJoonsoo Kim { 60f2ca0b55SJoonsoo Kim unsigned long entries[4]; 61af52bf6bSThomas Gleixner unsigned int nr_entries; 62f2ca0b55SJoonsoo Kim 63af52bf6bSThomas Gleixner nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 0); 64af52bf6bSThomas Gleixner return stack_depot_save(entries, nr_entries, GFP_KERNEL); 65dab4ead1SVlastimil Babka } 66dab4ead1SVlastimil Babka 67dab4ead1SVlastimil Babka static noinline void register_dummy_stack(void) 68dab4ead1SVlastimil Babka { 69dab4ead1SVlastimil Babka dummy_handle = create_dummy_stack(); 70f2ca0b55SJoonsoo Kim } 71f2ca0b55SJoonsoo Kim 72f2ca0b55SJoonsoo Kim static noinline void register_failure_stack(void) 73f2ca0b55SJoonsoo Kim { 74dab4ead1SVlastimil Babka failure_handle = create_dummy_stack(); 75dab4ead1SVlastimil Babka } 76f2ca0b55SJoonsoo Kim 77dab4ead1SVlastimil Babka static noinline void register_early_stack(void) 78dab4ead1SVlastimil Babka { 79dab4ead1SVlastimil Babka early_handle = create_dummy_stack(); 80f2ca0b55SJoonsoo Kim } 81f2ca0b55SJoonsoo Kim 8248c96a36SJoonsoo Kim static void init_page_owner(void) 8348c96a36SJoonsoo Kim { 8448c96a36SJoonsoo Kim if (page_owner_disabled) 8548c96a36SJoonsoo Kim return; 8648c96a36SJoonsoo Kim 87f2ca0b55SJoonsoo Kim register_dummy_stack(); 88f2ca0b55SJoonsoo Kim register_failure_stack(); 89dab4ead1SVlastimil Babka register_early_stack(); 907dd80b8aSVlastimil Babka static_branch_enable(&page_owner_inited); 9161cf5febSJoonsoo Kim init_early_allocated_pages(); 9248c96a36SJoonsoo Kim } 9348c96a36SJoonsoo Kim 9448c96a36SJoonsoo Kim struct page_ext_operations page_owner_ops = { 959300d8dfSJoonsoo Kim .size = sizeof(struct page_owner), 9648c96a36SJoonsoo Kim .need = need_page_owner, 9748c96a36SJoonsoo Kim .init = init_page_owner, 9848c96a36SJoonsoo Kim }; 9948c96a36SJoonsoo Kim 1009300d8dfSJoonsoo Kim static inline struct page_owner *get_page_owner(struct page_ext *page_ext) 1019300d8dfSJoonsoo Kim { 1029300d8dfSJoonsoo Kim return (void *)page_ext + page_owner_ops.offset; 1039300d8dfSJoonsoo Kim } 1049300d8dfSJoonsoo Kim 10548c96a36SJoonsoo Kim void __reset_page_owner(struct page *page, unsigned int order) 10648c96a36SJoonsoo Kim { 10748c96a36SJoonsoo Kim int i; 10848c96a36SJoonsoo Kim struct page_ext *page_ext; 10948c96a36SJoonsoo Kim 11048c96a36SJoonsoo Kim for (i = 0; i < (1 << order); i++) { 11148c96a36SJoonsoo Kim page_ext = lookup_page_ext(page + i); 112f86e4271SYang Shi if (unlikely(!page_ext)) 113f86e4271SYang Shi continue; 114*37389167SVlastimil Babka __clear_bit(PAGE_EXT_OWNER_ACTIVE, &page_ext->flags); 11548c96a36SJoonsoo Kim } 11648c96a36SJoonsoo Kim } 11748c96a36SJoonsoo Kim 118af52bf6bSThomas Gleixner static inline bool check_recursive_alloc(unsigned long *entries, 119af52bf6bSThomas Gleixner unsigned int nr_entries, 120f2ca0b55SJoonsoo Kim unsigned long ip) 12148c96a36SJoonsoo Kim { 122af52bf6bSThomas Gleixner unsigned int i; 123f86e4271SYang Shi 124af52bf6bSThomas Gleixner for (i = 0; i < nr_entries; i++) { 125af52bf6bSThomas Gleixner if (entries[i] == ip) 126f2ca0b55SJoonsoo Kim return true; 127f2ca0b55SJoonsoo Kim } 128f2ca0b55SJoonsoo Kim return false; 129f2ca0b55SJoonsoo Kim } 130f2ca0b55SJoonsoo Kim 131f2ca0b55SJoonsoo Kim static noinline depot_stack_handle_t save_stack(gfp_t flags) 132f2ca0b55SJoonsoo Kim { 133f2ca0b55SJoonsoo Kim unsigned long entries[PAGE_OWNER_STACK_DEPTH]; 134f2ca0b55SJoonsoo Kim depot_stack_handle_t handle; 135af52bf6bSThomas Gleixner unsigned int nr_entries; 136f2ca0b55SJoonsoo Kim 137af52bf6bSThomas Gleixner nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 2); 138f2ca0b55SJoonsoo Kim 139f2ca0b55SJoonsoo Kim /* 140af52bf6bSThomas Gleixner * We need to check recursion here because our request to 141af52bf6bSThomas Gleixner * stackdepot could trigger memory allocation to save new 142af52bf6bSThomas Gleixner * entry. New memory allocation would reach here and call 143af52bf6bSThomas Gleixner * stack_depot_save_entries() again if we don't catch it. There is 144af52bf6bSThomas Gleixner * still not enough memory in stackdepot so it would try to 145af52bf6bSThomas Gleixner * allocate memory again and loop forever. 146f2ca0b55SJoonsoo Kim */ 147af52bf6bSThomas Gleixner if (check_recursive_alloc(entries, nr_entries, _RET_IP_)) 148f2ca0b55SJoonsoo Kim return dummy_handle; 149f2ca0b55SJoonsoo Kim 150af52bf6bSThomas Gleixner handle = stack_depot_save(entries, nr_entries, flags); 151f2ca0b55SJoonsoo Kim if (!handle) 152f2ca0b55SJoonsoo Kim handle = failure_handle; 153f2ca0b55SJoonsoo Kim 154f2ca0b55SJoonsoo Kim return handle; 155f2ca0b55SJoonsoo Kim } 156f2ca0b55SJoonsoo Kim 1577e2f2a0cSVlastimil Babka static inline void __set_page_owner_handle(struct page *page, 1587e2f2a0cSVlastimil Babka struct page_ext *page_ext, depot_stack_handle_t handle, 1597e2f2a0cSVlastimil Babka unsigned int order, gfp_t gfp_mask) 160f2ca0b55SJoonsoo Kim { 1619300d8dfSJoonsoo Kim struct page_owner *page_owner; 1627e2f2a0cSVlastimil Babka int i; 16348c96a36SJoonsoo Kim 1647e2f2a0cSVlastimil Babka for (i = 0; i < (1 << order); i++) { 1659300d8dfSJoonsoo Kim page_owner = get_page_owner(page_ext); 166dab4ead1SVlastimil Babka page_owner->handle = handle; 1679300d8dfSJoonsoo Kim page_owner->order = order; 1689300d8dfSJoonsoo Kim page_owner->gfp_mask = gfp_mask; 1699300d8dfSJoonsoo Kim page_owner->last_migrate_reason = -1; 17048c96a36SJoonsoo Kim __set_bit(PAGE_EXT_OWNER, &page_ext->flags); 171*37389167SVlastimil Babka __set_bit(PAGE_EXT_OWNER_ACTIVE, &page_ext->flags); 1727e2f2a0cSVlastimil Babka 1737e2f2a0cSVlastimil Babka page_ext = lookup_page_ext(page + i); 1747e2f2a0cSVlastimil Babka } 17548c96a36SJoonsoo Kim } 17648c96a36SJoonsoo Kim 177dab4ead1SVlastimil Babka noinline void __set_page_owner(struct page *page, unsigned int order, 178dab4ead1SVlastimil Babka gfp_t gfp_mask) 179dab4ead1SVlastimil Babka { 180dab4ead1SVlastimil Babka struct page_ext *page_ext = lookup_page_ext(page); 181dab4ead1SVlastimil Babka depot_stack_handle_t handle; 182dab4ead1SVlastimil Babka 183dab4ead1SVlastimil Babka if (unlikely(!page_ext)) 184dab4ead1SVlastimil Babka return; 185dab4ead1SVlastimil Babka 186dab4ead1SVlastimil Babka handle = save_stack(gfp_mask); 1877e2f2a0cSVlastimil Babka __set_page_owner_handle(page, page_ext, handle, order, gfp_mask); 188dab4ead1SVlastimil Babka } 189dab4ead1SVlastimil Babka 1907cd12b4aSVlastimil Babka void __set_page_owner_migrate_reason(struct page *page, int reason) 1917cd12b4aSVlastimil Babka { 1927cd12b4aSVlastimil Babka struct page_ext *page_ext = lookup_page_ext(page); 1939300d8dfSJoonsoo Kim struct page_owner *page_owner; 1949300d8dfSJoonsoo Kim 195f86e4271SYang Shi if (unlikely(!page_ext)) 196f86e4271SYang Shi return; 1977cd12b4aSVlastimil Babka 1989300d8dfSJoonsoo Kim page_owner = get_page_owner(page_ext); 1999300d8dfSJoonsoo Kim page_owner->last_migrate_reason = reason; 2007cd12b4aSVlastimil Babka } 2017cd12b4aSVlastimil Babka 202a9627bc5SJoonsoo Kim void __split_page_owner(struct page *page, unsigned int order) 203e2cfc911SJoonsoo Kim { 204a9627bc5SJoonsoo Kim int i; 205e2cfc911SJoonsoo Kim struct page_ext *page_ext = lookup_page_ext(page); 2069300d8dfSJoonsoo Kim struct page_owner *page_owner; 207e2cfc911SJoonsoo Kim 208a9627bc5SJoonsoo Kim if (unlikely(!page_ext)) 209a9627bc5SJoonsoo Kim return; 210a9627bc5SJoonsoo Kim 2119300d8dfSJoonsoo Kim page_owner = get_page_owner(page_ext); 2129300d8dfSJoonsoo Kim page_owner->order = 0; 2137e2f2a0cSVlastimil Babka for (i = 1; i < (1 << order); i++) { 2147e2f2a0cSVlastimil Babka page_ext = lookup_page_ext(page + i); 2157e2f2a0cSVlastimil Babka page_owner = get_page_owner(page_ext); 2167e2f2a0cSVlastimil Babka page_owner->order = 0; 2177e2f2a0cSVlastimil Babka } 218e2cfc911SJoonsoo Kim } 219e2cfc911SJoonsoo Kim 220d435edcaSVlastimil Babka void __copy_page_owner(struct page *oldpage, struct page *newpage) 221d435edcaSVlastimil Babka { 222d435edcaSVlastimil Babka struct page_ext *old_ext = lookup_page_ext(oldpage); 223d435edcaSVlastimil Babka struct page_ext *new_ext = lookup_page_ext(newpage); 2249300d8dfSJoonsoo Kim struct page_owner *old_page_owner, *new_page_owner; 225d435edcaSVlastimil Babka 226f86e4271SYang Shi if (unlikely(!old_ext || !new_ext)) 227f86e4271SYang Shi return; 228f86e4271SYang Shi 2299300d8dfSJoonsoo Kim old_page_owner = get_page_owner(old_ext); 2309300d8dfSJoonsoo Kim new_page_owner = get_page_owner(new_ext); 2319300d8dfSJoonsoo Kim new_page_owner->order = old_page_owner->order; 2329300d8dfSJoonsoo Kim new_page_owner->gfp_mask = old_page_owner->gfp_mask; 2339300d8dfSJoonsoo Kim new_page_owner->last_migrate_reason = 2349300d8dfSJoonsoo Kim old_page_owner->last_migrate_reason; 2359300d8dfSJoonsoo Kim new_page_owner->handle = old_page_owner->handle; 236d435edcaSVlastimil Babka 237d435edcaSVlastimil Babka /* 238d435edcaSVlastimil Babka * We don't clear the bit on the oldpage as it's going to be freed 239d435edcaSVlastimil Babka * after migration. Until then, the info can be useful in case of 240d435edcaSVlastimil Babka * a bug, and the overal stats will be off a bit only temporarily. 241d435edcaSVlastimil Babka * Also, migrate_misplaced_transhuge_page() can still fail the 242d435edcaSVlastimil Babka * migration and then we want the oldpage to retain the info. But 243d435edcaSVlastimil Babka * in that case we also don't need to explicitly clear the info from 244d435edcaSVlastimil Babka * the new page, which will be freed. 245d435edcaSVlastimil Babka */ 246d435edcaSVlastimil Babka __set_bit(PAGE_EXT_OWNER, &new_ext->flags); 247*37389167SVlastimil Babka __set_bit(PAGE_EXT_OWNER_ACTIVE, &new_ext->flags); 248d435edcaSVlastimil Babka } 249d435edcaSVlastimil Babka 250e2f612e6SJoonsoo Kim void pagetypeinfo_showmixedcount_print(struct seq_file *m, 251e2f612e6SJoonsoo Kim pg_data_t *pgdat, struct zone *zone) 252e2f612e6SJoonsoo Kim { 253e2f612e6SJoonsoo Kim struct page *page; 254e2f612e6SJoonsoo Kim struct page_ext *page_ext; 2559300d8dfSJoonsoo Kim struct page_owner *page_owner; 256e2f612e6SJoonsoo Kim unsigned long pfn = zone->zone_start_pfn, block_end_pfn; 257e2f612e6SJoonsoo Kim unsigned long end_pfn = pfn + zone->spanned_pages; 258e2f612e6SJoonsoo Kim unsigned long count[MIGRATE_TYPES] = { 0, }; 259e2f612e6SJoonsoo Kim int pageblock_mt, page_mt; 260e2f612e6SJoonsoo Kim int i; 261e2f612e6SJoonsoo Kim 262e2f612e6SJoonsoo Kim /* Scan block by block. First and last block may be incomplete */ 263e2f612e6SJoonsoo Kim pfn = zone->zone_start_pfn; 264e2f612e6SJoonsoo Kim 265e2f612e6SJoonsoo Kim /* 266e2f612e6SJoonsoo Kim * Walk the zone in pageblock_nr_pages steps. If a page block spans 267e2f612e6SJoonsoo Kim * a zone boundary, it will be double counted between zones. This does 268e2f612e6SJoonsoo Kim * not matter as the mixed block count will still be correct 269e2f612e6SJoonsoo Kim */ 270e2f612e6SJoonsoo Kim for (; pfn < end_pfn; ) { 271e2f612e6SJoonsoo Kim if (!pfn_valid(pfn)) { 272e2f612e6SJoonsoo Kim pfn = ALIGN(pfn + 1, MAX_ORDER_NR_PAGES); 273e2f612e6SJoonsoo Kim continue; 274e2f612e6SJoonsoo Kim } 275e2f612e6SJoonsoo Kim 276e2f612e6SJoonsoo Kim block_end_pfn = ALIGN(pfn + 1, pageblock_nr_pages); 277e2f612e6SJoonsoo Kim block_end_pfn = min(block_end_pfn, end_pfn); 278e2f612e6SJoonsoo Kim 279e2f612e6SJoonsoo Kim page = pfn_to_page(pfn); 280e2f612e6SJoonsoo Kim pageblock_mt = get_pageblock_migratetype(page); 281e2f612e6SJoonsoo Kim 282e2f612e6SJoonsoo Kim for (; pfn < block_end_pfn; pfn++) { 283e2f612e6SJoonsoo Kim if (!pfn_valid_within(pfn)) 284e2f612e6SJoonsoo Kim continue; 285e2f612e6SJoonsoo Kim 286e2f612e6SJoonsoo Kim page = pfn_to_page(pfn); 287e2f612e6SJoonsoo Kim 288e2f612e6SJoonsoo Kim if (page_zone(page) != zone) 289e2f612e6SJoonsoo Kim continue; 290e2f612e6SJoonsoo Kim 291e2f612e6SJoonsoo Kim if (PageBuddy(page)) { 292727c080fSVinayak Menon unsigned long freepage_order; 293727c080fSVinayak Menon 294727c080fSVinayak Menon freepage_order = page_order_unsafe(page); 295727c080fSVinayak Menon if (freepage_order < MAX_ORDER) 296727c080fSVinayak Menon pfn += (1UL << freepage_order) - 1; 297e2f612e6SJoonsoo Kim continue; 298e2f612e6SJoonsoo Kim } 299e2f612e6SJoonsoo Kim 300e2f612e6SJoonsoo Kim if (PageReserved(page)) 301e2f612e6SJoonsoo Kim continue; 302e2f612e6SJoonsoo Kim 303e2f612e6SJoonsoo Kim page_ext = lookup_page_ext(page); 304e2f612e6SJoonsoo Kim if (unlikely(!page_ext)) 305e2f612e6SJoonsoo Kim continue; 306e2f612e6SJoonsoo Kim 307*37389167SVlastimil Babka if (!test_bit(PAGE_EXT_OWNER_ACTIVE, &page_ext->flags)) 308e2f612e6SJoonsoo Kim continue; 309e2f612e6SJoonsoo Kim 3109300d8dfSJoonsoo Kim page_owner = get_page_owner(page_ext); 3119300d8dfSJoonsoo Kim page_mt = gfpflags_to_migratetype( 3129300d8dfSJoonsoo Kim page_owner->gfp_mask); 313e2f612e6SJoonsoo Kim if (pageblock_mt != page_mt) { 314e2f612e6SJoonsoo Kim if (is_migrate_cma(pageblock_mt)) 315e2f612e6SJoonsoo Kim count[MIGRATE_MOVABLE]++; 316e2f612e6SJoonsoo Kim else 317e2f612e6SJoonsoo Kim count[pageblock_mt]++; 318e2f612e6SJoonsoo Kim 319e2f612e6SJoonsoo Kim pfn = block_end_pfn; 320e2f612e6SJoonsoo Kim break; 321e2f612e6SJoonsoo Kim } 3229300d8dfSJoonsoo Kim pfn += (1UL << page_owner->order) - 1; 323e2f612e6SJoonsoo Kim } 324e2f612e6SJoonsoo Kim } 325e2f612e6SJoonsoo Kim 326e2f612e6SJoonsoo Kim /* Print counts */ 327e2f612e6SJoonsoo Kim seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name); 328e2f612e6SJoonsoo Kim for (i = 0; i < MIGRATE_TYPES; i++) 329e2f612e6SJoonsoo Kim seq_printf(m, "%12lu ", count[i]); 330e2f612e6SJoonsoo Kim seq_putc(m, '\n'); 331e2f612e6SJoonsoo Kim } 332e2f612e6SJoonsoo Kim 33348c96a36SJoonsoo Kim static ssize_t 33448c96a36SJoonsoo Kim print_page_owner(char __user *buf, size_t count, unsigned long pfn, 3359300d8dfSJoonsoo Kim struct page *page, struct page_owner *page_owner, 336f2ca0b55SJoonsoo Kim depot_stack_handle_t handle) 33748c96a36SJoonsoo Kim { 338af52bf6bSThomas Gleixner int ret, pageblock_mt, page_mt; 339af52bf6bSThomas Gleixner unsigned long *entries; 340af52bf6bSThomas Gleixner unsigned int nr_entries; 34148c96a36SJoonsoo Kim char *kbuf; 34248c96a36SJoonsoo Kim 343c8f61cfcSMiles Chen count = min_t(size_t, count, PAGE_SIZE); 34448c96a36SJoonsoo Kim kbuf = kmalloc(count, GFP_KERNEL); 34548c96a36SJoonsoo Kim if (!kbuf) 34648c96a36SJoonsoo Kim return -ENOMEM; 34748c96a36SJoonsoo Kim 34848c96a36SJoonsoo Kim ret = snprintf(kbuf, count, 34960f30350SVlastimil Babka "Page allocated via order %u, mask %#x(%pGg)\n", 3509300d8dfSJoonsoo Kim page_owner->order, page_owner->gfp_mask, 3519300d8dfSJoonsoo Kim &page_owner->gfp_mask); 35248c96a36SJoonsoo Kim 35348c96a36SJoonsoo Kim if (ret >= count) 35448c96a36SJoonsoo Kim goto err; 35548c96a36SJoonsoo Kim 35648c96a36SJoonsoo Kim /* Print information relevant to grouping pages by mobility */ 3570b423ca2SMel Gorman pageblock_mt = get_pageblock_migratetype(page); 3589300d8dfSJoonsoo Kim page_mt = gfpflags_to_migratetype(page_owner->gfp_mask); 35948c96a36SJoonsoo Kim ret += snprintf(kbuf + ret, count - ret, 36060f30350SVlastimil Babka "PFN %lu type %s Block %lu type %s Flags %#lx(%pGp)\n", 36148c96a36SJoonsoo Kim pfn, 36260f30350SVlastimil Babka migratetype_names[page_mt], 36348c96a36SJoonsoo Kim pfn >> pageblock_order, 36460f30350SVlastimil Babka migratetype_names[pageblock_mt], 36560f30350SVlastimil Babka page->flags, &page->flags); 36648c96a36SJoonsoo Kim 36748c96a36SJoonsoo Kim if (ret >= count) 36848c96a36SJoonsoo Kim goto err; 36948c96a36SJoonsoo Kim 370af52bf6bSThomas Gleixner nr_entries = stack_depot_fetch(handle, &entries); 371af52bf6bSThomas Gleixner ret += stack_trace_snprint(kbuf + ret, count - ret, entries, nr_entries, 0); 37248c96a36SJoonsoo Kim if (ret >= count) 37348c96a36SJoonsoo Kim goto err; 37448c96a36SJoonsoo Kim 3759300d8dfSJoonsoo Kim if (page_owner->last_migrate_reason != -1) { 3767cd12b4aSVlastimil Babka ret += snprintf(kbuf + ret, count - ret, 3777cd12b4aSVlastimil Babka "Page has been migrated, last migrate reason: %s\n", 3789300d8dfSJoonsoo Kim migrate_reason_names[page_owner->last_migrate_reason]); 3797cd12b4aSVlastimil Babka if (ret >= count) 3807cd12b4aSVlastimil Babka goto err; 3817cd12b4aSVlastimil Babka } 3827cd12b4aSVlastimil Babka 38348c96a36SJoonsoo Kim ret += snprintf(kbuf + ret, count - ret, "\n"); 38448c96a36SJoonsoo Kim if (ret >= count) 38548c96a36SJoonsoo Kim goto err; 38648c96a36SJoonsoo Kim 38748c96a36SJoonsoo Kim if (copy_to_user(buf, kbuf, ret)) 38848c96a36SJoonsoo Kim ret = -EFAULT; 38948c96a36SJoonsoo Kim 39048c96a36SJoonsoo Kim kfree(kbuf); 39148c96a36SJoonsoo Kim return ret; 39248c96a36SJoonsoo Kim 39348c96a36SJoonsoo Kim err: 39448c96a36SJoonsoo Kim kfree(kbuf); 39548c96a36SJoonsoo Kim return -ENOMEM; 39648c96a36SJoonsoo Kim } 39748c96a36SJoonsoo Kim 3984e462112SVlastimil Babka void __dump_page_owner(struct page *page) 3994e462112SVlastimil Babka { 4004e462112SVlastimil Babka struct page_ext *page_ext = lookup_page_ext(page); 4019300d8dfSJoonsoo Kim struct page_owner *page_owner; 402f2ca0b55SJoonsoo Kim depot_stack_handle_t handle; 403af52bf6bSThomas Gleixner unsigned long *entries; 404af52bf6bSThomas Gleixner unsigned int nr_entries; 4058285027fSSudip Mukherjee gfp_t gfp_mask; 4068285027fSSudip Mukherjee int mt; 4074e462112SVlastimil Babka 408f86e4271SYang Shi if (unlikely(!page_ext)) { 409f86e4271SYang Shi pr_alert("There is not page extension available.\n"); 410f86e4271SYang Shi return; 411f86e4271SYang Shi } 4129300d8dfSJoonsoo Kim 4139300d8dfSJoonsoo Kim page_owner = get_page_owner(page_ext); 4149300d8dfSJoonsoo Kim gfp_mask = page_owner->gfp_mask; 4158285027fSSudip Mukherjee mt = gfpflags_to_migratetype(gfp_mask); 416f86e4271SYang Shi 4174e462112SVlastimil Babka if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags)) { 418*37389167SVlastimil Babka pr_alert("page_owner info is not present (never set?)\n"); 4194e462112SVlastimil Babka return; 4204e462112SVlastimil Babka } 4214e462112SVlastimil Babka 422*37389167SVlastimil Babka if (test_bit(PAGE_EXT_OWNER_ACTIVE, &page_ext->flags)) 423*37389167SVlastimil Babka pr_alert("page_owner tracks the page as allocated\n"); 424*37389167SVlastimil Babka else 425*37389167SVlastimil Babka pr_alert("page_owner tracks the page as freed\n"); 426*37389167SVlastimil Babka 427*37389167SVlastimil Babka pr_alert("page last allocated via order %u, migratetype %s, gfp_mask %#x(%pGg)\n", 428*37389167SVlastimil Babka page_owner->order, migratetype_names[mt], gfp_mask, &gfp_mask); 429*37389167SVlastimil Babka 4309300d8dfSJoonsoo Kim handle = READ_ONCE(page_owner->handle); 431f2ca0b55SJoonsoo Kim if (!handle) { 432*37389167SVlastimil Babka pr_alert("page_owner allocation stack trace missing\n"); 433*37389167SVlastimil Babka } else { 434af52bf6bSThomas Gleixner nr_entries = stack_depot_fetch(handle, &entries); 435af52bf6bSThomas Gleixner stack_trace_print(entries, nr_entries, 0); 436*37389167SVlastimil Babka } 4374e462112SVlastimil Babka 4389300d8dfSJoonsoo Kim if (page_owner->last_migrate_reason != -1) 4394e462112SVlastimil Babka pr_alert("page has been migrated, last migrate reason: %s\n", 4409300d8dfSJoonsoo Kim migrate_reason_names[page_owner->last_migrate_reason]); 4414e462112SVlastimil Babka } 4424e462112SVlastimil Babka 44348c96a36SJoonsoo Kim static ssize_t 44448c96a36SJoonsoo Kim read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos) 44548c96a36SJoonsoo Kim { 44648c96a36SJoonsoo Kim unsigned long pfn; 44748c96a36SJoonsoo Kim struct page *page; 44848c96a36SJoonsoo Kim struct page_ext *page_ext; 4499300d8dfSJoonsoo Kim struct page_owner *page_owner; 450f2ca0b55SJoonsoo Kim depot_stack_handle_t handle; 45148c96a36SJoonsoo Kim 4527dd80b8aSVlastimil Babka if (!static_branch_unlikely(&page_owner_inited)) 45348c96a36SJoonsoo Kim return -EINVAL; 45448c96a36SJoonsoo Kim 45548c96a36SJoonsoo Kim page = NULL; 45648c96a36SJoonsoo Kim pfn = min_low_pfn + *ppos; 45748c96a36SJoonsoo Kim 45848c96a36SJoonsoo Kim /* Find a valid PFN or the start of a MAX_ORDER_NR_PAGES area */ 45948c96a36SJoonsoo Kim while (!pfn_valid(pfn) && (pfn & (MAX_ORDER_NR_PAGES - 1)) != 0) 46048c96a36SJoonsoo Kim pfn++; 46148c96a36SJoonsoo Kim 46248c96a36SJoonsoo Kim drain_all_pages(NULL); 46348c96a36SJoonsoo Kim 46448c96a36SJoonsoo Kim /* Find an allocated page */ 46548c96a36SJoonsoo Kim for (; pfn < max_pfn; pfn++) { 46648c96a36SJoonsoo Kim /* 46748c96a36SJoonsoo Kim * If the new page is in a new MAX_ORDER_NR_PAGES area, 46848c96a36SJoonsoo Kim * validate the area as existing, skip it if not 46948c96a36SJoonsoo Kim */ 47048c96a36SJoonsoo Kim if ((pfn & (MAX_ORDER_NR_PAGES - 1)) == 0 && !pfn_valid(pfn)) { 47148c96a36SJoonsoo Kim pfn += MAX_ORDER_NR_PAGES - 1; 47248c96a36SJoonsoo Kim continue; 47348c96a36SJoonsoo Kim } 47448c96a36SJoonsoo Kim 47548c96a36SJoonsoo Kim /* Check for holes within a MAX_ORDER area */ 47648c96a36SJoonsoo Kim if (!pfn_valid_within(pfn)) 47748c96a36SJoonsoo Kim continue; 47848c96a36SJoonsoo Kim 47948c96a36SJoonsoo Kim page = pfn_to_page(pfn); 48048c96a36SJoonsoo Kim if (PageBuddy(page)) { 48148c96a36SJoonsoo Kim unsigned long freepage_order = page_order_unsafe(page); 48248c96a36SJoonsoo Kim 48348c96a36SJoonsoo Kim if (freepage_order < MAX_ORDER) 48448c96a36SJoonsoo Kim pfn += (1UL << freepage_order) - 1; 48548c96a36SJoonsoo Kim continue; 48648c96a36SJoonsoo Kim } 48748c96a36SJoonsoo Kim 48848c96a36SJoonsoo Kim page_ext = lookup_page_ext(page); 489f86e4271SYang Shi if (unlikely(!page_ext)) 490f86e4271SYang Shi continue; 49148c96a36SJoonsoo Kim 49248c96a36SJoonsoo Kim /* 49361cf5febSJoonsoo Kim * Some pages could be missed by concurrent allocation or free, 49461cf5febSJoonsoo Kim * because we don't hold the zone lock. 49548c96a36SJoonsoo Kim */ 49648c96a36SJoonsoo Kim if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags)) 49748c96a36SJoonsoo Kim continue; 49848c96a36SJoonsoo Kim 499*37389167SVlastimil Babka /* 500*37389167SVlastimil Babka * Although we do have the info about past allocation of free 501*37389167SVlastimil Babka * pages, it's not relevant for current memory usage. 502*37389167SVlastimil Babka */ 503*37389167SVlastimil Babka if (!test_bit(PAGE_EXT_OWNER_ACTIVE, &page_ext->flags)) 504*37389167SVlastimil Babka continue; 505*37389167SVlastimil Babka 5069300d8dfSJoonsoo Kim page_owner = get_page_owner(page_ext); 5079300d8dfSJoonsoo Kim 508f2ca0b55SJoonsoo Kim /* 5097e2f2a0cSVlastimil Babka * Don't print "tail" pages of high-order allocations as that 5107e2f2a0cSVlastimil Babka * would inflate the stats. 5117e2f2a0cSVlastimil Babka */ 5127e2f2a0cSVlastimil Babka if (!IS_ALIGNED(pfn, 1 << page_owner->order)) 5137e2f2a0cSVlastimil Babka continue; 5147e2f2a0cSVlastimil Babka 5157e2f2a0cSVlastimil Babka /* 516f2ca0b55SJoonsoo Kim * Access to page_ext->handle isn't synchronous so we should 517f2ca0b55SJoonsoo Kim * be careful to access it. 518f2ca0b55SJoonsoo Kim */ 5199300d8dfSJoonsoo Kim handle = READ_ONCE(page_owner->handle); 520f2ca0b55SJoonsoo Kim if (!handle) 521f2ca0b55SJoonsoo Kim continue; 522f2ca0b55SJoonsoo Kim 52348c96a36SJoonsoo Kim /* Record the next PFN to read in the file offset */ 52448c96a36SJoonsoo Kim *ppos = (pfn - min_low_pfn) + 1; 52548c96a36SJoonsoo Kim 526f2ca0b55SJoonsoo Kim return print_page_owner(buf, count, pfn, page, 5279300d8dfSJoonsoo Kim page_owner, handle); 52848c96a36SJoonsoo Kim } 52948c96a36SJoonsoo Kim 53048c96a36SJoonsoo Kim return 0; 53148c96a36SJoonsoo Kim } 53248c96a36SJoonsoo Kim 53361cf5febSJoonsoo Kim static void init_pages_in_zone(pg_data_t *pgdat, struct zone *zone) 53461cf5febSJoonsoo Kim { 5356787c1daSOscar Salvador unsigned long pfn = zone->zone_start_pfn; 5366787c1daSOscar Salvador unsigned long end_pfn = zone_end_pfn(zone); 53761cf5febSJoonsoo Kim unsigned long count = 0; 53861cf5febSJoonsoo Kim 53961cf5febSJoonsoo Kim /* 54061cf5febSJoonsoo Kim * Walk the zone in pageblock_nr_pages steps. If a page block spans 54161cf5febSJoonsoo Kim * a zone boundary, it will be double counted between zones. This does 54261cf5febSJoonsoo Kim * not matter as the mixed block count will still be correct 54361cf5febSJoonsoo Kim */ 54461cf5febSJoonsoo Kim for (; pfn < end_pfn; ) { 5456787c1daSOscar Salvador unsigned long block_end_pfn; 5466787c1daSOscar Salvador 54761cf5febSJoonsoo Kim if (!pfn_valid(pfn)) { 54861cf5febSJoonsoo Kim pfn = ALIGN(pfn + 1, MAX_ORDER_NR_PAGES); 54961cf5febSJoonsoo Kim continue; 55061cf5febSJoonsoo Kim } 55161cf5febSJoonsoo Kim 55261cf5febSJoonsoo Kim block_end_pfn = ALIGN(pfn + 1, pageblock_nr_pages); 55361cf5febSJoonsoo Kim block_end_pfn = min(block_end_pfn, end_pfn); 55461cf5febSJoonsoo Kim 55561cf5febSJoonsoo Kim for (; pfn < block_end_pfn; pfn++) { 5566787c1daSOscar Salvador struct page *page; 5576787c1daSOscar Salvador struct page_ext *page_ext; 5586787c1daSOscar Salvador 55961cf5febSJoonsoo Kim if (!pfn_valid_within(pfn)) 56061cf5febSJoonsoo Kim continue; 56161cf5febSJoonsoo Kim 56261cf5febSJoonsoo Kim page = pfn_to_page(pfn); 56361cf5febSJoonsoo Kim 5649d43f5aeSJoonsoo Kim if (page_zone(page) != zone) 5659d43f5aeSJoonsoo Kim continue; 5669d43f5aeSJoonsoo Kim 56761cf5febSJoonsoo Kim /* 56810903027SVlastimil Babka * To avoid having to grab zone->lock, be a little 56910903027SVlastimil Babka * careful when reading buddy page order. The only 57010903027SVlastimil Babka * danger is that we skip too much and potentially miss 57110903027SVlastimil Babka * some early allocated pages, which is better than 57210903027SVlastimil Babka * heavy lock contention. 57361cf5febSJoonsoo Kim */ 57461cf5febSJoonsoo Kim if (PageBuddy(page)) { 57510903027SVlastimil Babka unsigned long order = page_order_unsafe(page); 57610903027SVlastimil Babka 57710903027SVlastimil Babka if (order > 0 && order < MAX_ORDER) 57810903027SVlastimil Babka pfn += (1UL << order) - 1; 57961cf5febSJoonsoo Kim continue; 58061cf5febSJoonsoo Kim } 58161cf5febSJoonsoo Kim 58261cf5febSJoonsoo Kim if (PageReserved(page)) 58361cf5febSJoonsoo Kim continue; 58461cf5febSJoonsoo Kim 58561cf5febSJoonsoo Kim page_ext = lookup_page_ext(page); 586f86e4271SYang Shi if (unlikely(!page_ext)) 587f86e4271SYang Shi continue; 58861cf5febSJoonsoo Kim 589dab4ead1SVlastimil Babka /* Maybe overlapping zone */ 59061cf5febSJoonsoo Kim if (test_bit(PAGE_EXT_OWNER, &page_ext->flags)) 59161cf5febSJoonsoo Kim continue; 59261cf5febSJoonsoo Kim 59361cf5febSJoonsoo Kim /* Found early allocated page */ 5947e2f2a0cSVlastimil Babka __set_page_owner_handle(page, page_ext, early_handle, 5957e2f2a0cSVlastimil Babka 0, 0); 59661cf5febSJoonsoo Kim count++; 59761cf5febSJoonsoo Kim } 59810903027SVlastimil Babka cond_resched(); 59961cf5febSJoonsoo Kim } 60061cf5febSJoonsoo Kim 60161cf5febSJoonsoo Kim pr_info("Node %d, zone %8s: page owner found early allocated %lu pages\n", 60261cf5febSJoonsoo Kim pgdat->node_id, zone->name, count); 60361cf5febSJoonsoo Kim } 60461cf5febSJoonsoo Kim 60561cf5febSJoonsoo Kim static void init_zones_in_node(pg_data_t *pgdat) 60661cf5febSJoonsoo Kim { 60761cf5febSJoonsoo Kim struct zone *zone; 60861cf5febSJoonsoo Kim struct zone *node_zones = pgdat->node_zones; 60961cf5febSJoonsoo Kim 61061cf5febSJoonsoo Kim for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) { 61161cf5febSJoonsoo Kim if (!populated_zone(zone)) 61261cf5febSJoonsoo Kim continue; 61361cf5febSJoonsoo Kim 61461cf5febSJoonsoo Kim init_pages_in_zone(pgdat, zone); 61561cf5febSJoonsoo Kim } 61661cf5febSJoonsoo Kim } 61761cf5febSJoonsoo Kim 61861cf5febSJoonsoo Kim static void init_early_allocated_pages(void) 61961cf5febSJoonsoo Kim { 62061cf5febSJoonsoo Kim pg_data_t *pgdat; 62161cf5febSJoonsoo Kim 62261cf5febSJoonsoo Kim for_each_online_pgdat(pgdat) 62361cf5febSJoonsoo Kim init_zones_in_node(pgdat); 62461cf5febSJoonsoo Kim } 62561cf5febSJoonsoo Kim 62648c96a36SJoonsoo Kim static const struct file_operations proc_page_owner_operations = { 62748c96a36SJoonsoo Kim .read = read_page_owner, 62848c96a36SJoonsoo Kim }; 62948c96a36SJoonsoo Kim 63048c96a36SJoonsoo Kim static int __init pageowner_init(void) 63148c96a36SJoonsoo Kim { 6327dd80b8aSVlastimil Babka if (!static_branch_unlikely(&page_owner_inited)) { 63348c96a36SJoonsoo Kim pr_info("page_owner is disabled\n"); 63448c96a36SJoonsoo Kim return 0; 63548c96a36SJoonsoo Kim } 63648c96a36SJoonsoo Kim 637d9f7979cSGreg Kroah-Hartman debugfs_create_file("page_owner", 0400, NULL, NULL, 638d9f7979cSGreg Kroah-Hartman &proc_page_owner_operations); 63948c96a36SJoonsoo Kim 640d9f7979cSGreg Kroah-Hartman return 0; 64148c96a36SJoonsoo Kim } 64244c5af96SPaul Gortmaker late_initcall(pageowner_init) 643