1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 248c96a36SJoonsoo Kim #include <linux/debugfs.h> 348c96a36SJoonsoo Kim #include <linux/mm.h> 448c96a36SJoonsoo Kim #include <linux/slab.h> 548c96a36SJoonsoo Kim #include <linux/uaccess.h> 657c8a661SMike Rapoport #include <linux/memblock.h> 748c96a36SJoonsoo Kim #include <linux/stacktrace.h> 848c96a36SJoonsoo Kim #include <linux/page_owner.h> 97dd80b8aSVlastimil Babka #include <linux/jump_label.h> 107cd12b4aSVlastimil Babka #include <linux/migrate.h> 11f2ca0b55SJoonsoo Kim #include <linux/stackdepot.h> 12e2f612e6SJoonsoo Kim #include <linux/seq_file.h> 13fcf89358SWaiman Long #include <linux/memcontrol.h> 149cc7e96aSLiam Mark #include <linux/sched/clock.h> 15f2ca0b55SJoonsoo Kim 1648c96a36SJoonsoo Kim #include "internal.h" 1748c96a36SJoonsoo Kim 18f2ca0b55SJoonsoo Kim /* 19f2ca0b55SJoonsoo Kim * TODO: teach PAGE_OWNER_STACK_DEPTH (__dump_page_owner and save_stack) 20f2ca0b55SJoonsoo Kim * to use off stack temporal storage 21f2ca0b55SJoonsoo Kim */ 22f2ca0b55SJoonsoo Kim #define PAGE_OWNER_STACK_DEPTH (16) 23f2ca0b55SJoonsoo Kim 249300d8dfSJoonsoo Kim struct page_owner { 256b4c54e3SAyush Mittal unsigned short order; 266b4c54e3SAyush Mittal short last_migrate_reason; 279300d8dfSJoonsoo Kim gfp_t gfp_mask; 289300d8dfSJoonsoo Kim depot_stack_handle_t handle; 298974558fSVlastimil Babka depot_stack_handle_t free_handle; 309cc7e96aSLiam Mark u64 ts_nsec; 31866b4852SGeorgi Djakov u64 free_ts_nsec; 32865ed6a3SWaiman Long char comm[TASK_COMM_LEN]; 339cc7e96aSLiam Mark pid_t pid; 34bf215eabSYixuan Cao pid_t tgid; 351b5c65b6SBarry Song pid_t free_pid; 361b5c65b6SBarry Song pid_t free_tgid; 379300d8dfSJoonsoo Kim }; 389300d8dfSJoonsoo Kim 393645b5ecSFanjun Kong static bool page_owner_enabled __initdata; 407dd80b8aSVlastimil Babka DEFINE_STATIC_KEY_FALSE(page_owner_inited); 4148c96a36SJoonsoo Kim 42f2ca0b55SJoonsoo Kim static depot_stack_handle_t dummy_handle; 43f2ca0b55SJoonsoo Kim static depot_stack_handle_t failure_handle; 44dab4ead1SVlastimil Babka static depot_stack_handle_t early_handle; 45f2ca0b55SJoonsoo Kim 4661cf5febSJoonsoo Kim static void init_early_allocated_pages(void); 4761cf5febSJoonsoo Kim 481173194eSDou Liyang static int __init early_page_owner_param(char *buf) 4948c96a36SJoonsoo Kim { 50a5f1783bSVlastimil Babka int ret = kstrtobool(buf, &page_owner_enabled); 51a5f1783bSVlastimil Babka 52a5f1783bSVlastimil Babka if (page_owner_enabled) 531c0310adSAndrey Konovalov stack_depot_request_early_init(); 54a5f1783bSVlastimil Babka 55a5f1783bSVlastimil Babka return ret; 5648c96a36SJoonsoo Kim } 5748c96a36SJoonsoo Kim early_param("page_owner", early_page_owner_param); 5848c96a36SJoonsoo Kim 59cab0a7c1STing Liu static __init bool need_page_owner(void) 6048c96a36SJoonsoo Kim { 610fe9a448SVlastimil Babka return page_owner_enabled; 6248c96a36SJoonsoo Kim } 6348c96a36SJoonsoo Kim 64dab4ead1SVlastimil Babka static __always_inline depot_stack_handle_t create_dummy_stack(void) 65f2ca0b55SJoonsoo Kim { 66f2ca0b55SJoonsoo Kim unsigned long entries[4]; 67af52bf6bSThomas Gleixner unsigned int nr_entries; 68f2ca0b55SJoonsoo Kim 69af52bf6bSThomas Gleixner nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 0); 70af52bf6bSThomas Gleixner return stack_depot_save(entries, nr_entries, GFP_KERNEL); 71dab4ead1SVlastimil Babka } 72dab4ead1SVlastimil Babka 73dab4ead1SVlastimil Babka static noinline void register_dummy_stack(void) 74dab4ead1SVlastimil Babka { 75dab4ead1SVlastimil Babka dummy_handle = create_dummy_stack(); 76f2ca0b55SJoonsoo Kim } 77f2ca0b55SJoonsoo Kim 78f2ca0b55SJoonsoo Kim static noinline void register_failure_stack(void) 79f2ca0b55SJoonsoo Kim { 80dab4ead1SVlastimil Babka failure_handle = create_dummy_stack(); 81dab4ead1SVlastimil Babka } 82f2ca0b55SJoonsoo Kim 83dab4ead1SVlastimil Babka static noinline void register_early_stack(void) 84dab4ead1SVlastimil Babka { 85dab4ead1SVlastimil Babka early_handle = create_dummy_stack(); 86f2ca0b55SJoonsoo Kim } 87f2ca0b55SJoonsoo Kim 88cab0a7c1STing Liu static __init void init_page_owner(void) 8948c96a36SJoonsoo Kim { 900fe9a448SVlastimil Babka if (!page_owner_enabled) 9148c96a36SJoonsoo Kim return; 9248c96a36SJoonsoo Kim 93f2ca0b55SJoonsoo Kim register_dummy_stack(); 94f2ca0b55SJoonsoo Kim register_failure_stack(); 95dab4ead1SVlastimil Babka register_early_stack(); 967dd80b8aSVlastimil Babka static_branch_enable(&page_owner_inited); 9761cf5febSJoonsoo Kim init_early_allocated_pages(); 9848c96a36SJoonsoo Kim } 9948c96a36SJoonsoo Kim 10048c96a36SJoonsoo Kim struct page_ext_operations page_owner_ops = { 1019300d8dfSJoonsoo Kim .size = sizeof(struct page_owner), 10248c96a36SJoonsoo Kim .need = need_page_owner, 10348c96a36SJoonsoo Kim .init = init_page_owner, 1046189eb82SPasha Tatashin .need_shared_flags = true, 10548c96a36SJoonsoo Kim }; 10648c96a36SJoonsoo Kim 1079300d8dfSJoonsoo Kim static inline struct page_owner *get_page_owner(struct page_ext *page_ext) 1089300d8dfSJoonsoo Kim { 1091cac4c07SKemeng Shi return page_ext_data(page_ext, &page_owner_ops); 1109300d8dfSJoonsoo Kim } 1119300d8dfSJoonsoo Kim 112f2ca0b55SJoonsoo Kim static noinline depot_stack_handle_t save_stack(gfp_t flags) 113f2ca0b55SJoonsoo Kim { 114f2ca0b55SJoonsoo Kim unsigned long entries[PAGE_OWNER_STACK_DEPTH]; 115f2ca0b55SJoonsoo Kim depot_stack_handle_t handle; 116af52bf6bSThomas Gleixner unsigned int nr_entries; 117f2ca0b55SJoonsoo Kim 118f2ca0b55SJoonsoo Kim /* 1198e9b16c4SSergei Trofimovich * Avoid recursion. 1208e9b16c4SSergei Trofimovich * 1218e9b16c4SSergei Trofimovich * Sometimes page metadata allocation tracking requires more 1228e9b16c4SSergei Trofimovich * memory to be allocated: 1238e9b16c4SSergei Trofimovich * - when new stack trace is saved to stack depot 124f2ca0b55SJoonsoo Kim */ 1258e9b16c4SSergei Trofimovich if (current->in_page_owner) 126f2ca0b55SJoonsoo Kim return dummy_handle; 1278e9b16c4SSergei Trofimovich current->in_page_owner = 1; 128f2ca0b55SJoonsoo Kim 1298e9b16c4SSergei Trofimovich nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 2); 130af52bf6bSThomas Gleixner handle = stack_depot_save(entries, nr_entries, flags); 131f2ca0b55SJoonsoo Kim if (!handle) 132f2ca0b55SJoonsoo Kim handle = failure_handle; 133f2ca0b55SJoonsoo Kim 1348e9b16c4SSergei Trofimovich current->in_page_owner = 0; 135f2ca0b55SJoonsoo Kim return handle; 136f2ca0b55SJoonsoo Kim } 137f2ca0b55SJoonsoo Kim 1380093de69SYixuan Cao void __reset_page_owner(struct page *page, unsigned short order) 1398974558fSVlastimil Babka { 1408974558fSVlastimil Babka int i; 1418974558fSVlastimil Babka struct page_ext *page_ext; 142fab765c2SSergei Trofimovich depot_stack_handle_t handle; 1438974558fSVlastimil Babka struct page_owner *page_owner; 144866b4852SGeorgi Djakov u64 free_ts_nsec = local_clock(); 1458974558fSVlastimil Babka 146b1d5488aSCharan Teja Kalla page_ext = page_ext_get(page); 1478974558fSVlastimil Babka if (unlikely(!page_ext)) 1485556cfe8SVlastimil Babka return; 149fab765c2SSergei Trofimovich 150fab765c2SSergei Trofimovich handle = save_stack(GFP_NOWAIT | __GFP_NOWARN); 1515556cfe8SVlastimil Babka for (i = 0; i < (1 << order); i++) { 152fdf3bf80SVlastimil Babka __clear_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags); 1538974558fSVlastimil Babka page_owner = get_page_owner(page_ext); 1548974558fSVlastimil Babka page_owner->free_handle = handle; 155866b4852SGeorgi Djakov page_owner->free_ts_nsec = free_ts_nsec; 1561b5c65b6SBarry Song page_owner->free_pid = current->pid; 1571b5c65b6SBarry Song page_owner->free_tgid = current->tgid; 1585556cfe8SVlastimil Babka page_ext = page_ext_next(page_ext); 1598974558fSVlastimil Babka } 160b1d5488aSCharan Teja Kalla page_ext_put(page_ext); 1618974558fSVlastimil Babka } 1628974558fSVlastimil Babka 16364ea78d2Szhongjiang-ali static inline void __set_page_owner_handle(struct page_ext *page_ext, 16464ea78d2Szhongjiang-ali depot_stack_handle_t handle, 1650093de69SYixuan Cao unsigned short order, gfp_t gfp_mask) 166f2ca0b55SJoonsoo Kim { 1679300d8dfSJoonsoo Kim struct page_owner *page_owner; 1687e2f2a0cSVlastimil Babka int i; 16905a42199SHyeonggon Yoo u64 ts_nsec = local_clock(); 17048c96a36SJoonsoo Kim 1717e2f2a0cSVlastimil Babka for (i = 0; i < (1 << order); i++) { 1729300d8dfSJoonsoo Kim page_owner = get_page_owner(page_ext); 173dab4ead1SVlastimil Babka page_owner->handle = handle; 1749300d8dfSJoonsoo Kim page_owner->order = order; 1759300d8dfSJoonsoo Kim page_owner->gfp_mask = gfp_mask; 1769300d8dfSJoonsoo Kim page_owner->last_migrate_reason = -1; 1779cc7e96aSLiam Mark page_owner->pid = current->pid; 178bf215eabSYixuan Cao page_owner->tgid = current->tgid; 17905a42199SHyeonggon Yoo page_owner->ts_nsec = ts_nsec; 180cd8c1fd8SEric Dumazet strscpy(page_owner->comm, current->comm, 181865ed6a3SWaiman Long sizeof(page_owner->comm)); 18248c96a36SJoonsoo Kim __set_bit(PAGE_EXT_OWNER, &page_ext->flags); 183fdf3bf80SVlastimil Babka __set_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags); 1847e2f2a0cSVlastimil Babka 1855556cfe8SVlastimil Babka page_ext = page_ext_next(page_ext); 1867e2f2a0cSVlastimil Babka } 18748c96a36SJoonsoo Kim } 18848c96a36SJoonsoo Kim 1890093de69SYixuan Cao noinline void __set_page_owner(struct page *page, unsigned short order, 190dab4ead1SVlastimil Babka gfp_t gfp_mask) 191dab4ead1SVlastimil Babka { 192b1d5488aSCharan Teja Kalla struct page_ext *page_ext; 193dab4ead1SVlastimil Babka depot_stack_handle_t handle; 194dab4ead1SVlastimil Babka 195b1d5488aSCharan Teja Kalla handle = save_stack(gfp_mask); 196b1d5488aSCharan Teja Kalla 197b1d5488aSCharan Teja Kalla page_ext = page_ext_get(page); 198dab4ead1SVlastimil Babka if (unlikely(!page_ext)) 199dab4ead1SVlastimil Babka return; 20064ea78d2Szhongjiang-ali __set_page_owner_handle(page_ext, handle, order, gfp_mask); 201b1d5488aSCharan Teja Kalla page_ext_put(page_ext); 202dab4ead1SVlastimil Babka } 203dab4ead1SVlastimil Babka 2047cd12b4aSVlastimil Babka void __set_page_owner_migrate_reason(struct page *page, int reason) 2057cd12b4aSVlastimil Babka { 206b1d5488aSCharan Teja Kalla struct page_ext *page_ext = page_ext_get(page); 2079300d8dfSJoonsoo Kim struct page_owner *page_owner; 2089300d8dfSJoonsoo Kim 209f86e4271SYang Shi if (unlikely(!page_ext)) 210f86e4271SYang Shi return; 2117cd12b4aSVlastimil Babka 2129300d8dfSJoonsoo Kim page_owner = get_page_owner(page_ext); 2139300d8dfSJoonsoo Kim page_owner->last_migrate_reason = reason; 214b1d5488aSCharan Teja Kalla page_ext_put(page_ext); 2157cd12b4aSVlastimil Babka } 2167cd12b4aSVlastimil Babka 2178fb156c9SMatthew Wilcox (Oracle) void __split_page_owner(struct page *page, unsigned int nr) 218e2cfc911SJoonsoo Kim { 219a9627bc5SJoonsoo Kim int i; 220b1d5488aSCharan Teja Kalla struct page_ext *page_ext = page_ext_get(page); 2219300d8dfSJoonsoo Kim struct page_owner *page_owner; 222e2cfc911SJoonsoo Kim 223a9627bc5SJoonsoo Kim if (unlikely(!page_ext)) 224a9627bc5SJoonsoo Kim return; 225a9627bc5SJoonsoo Kim 2268fb156c9SMatthew Wilcox (Oracle) for (i = 0; i < nr; i++) { 2279300d8dfSJoonsoo Kim page_owner = get_page_owner(page_ext); 2289300d8dfSJoonsoo Kim page_owner->order = 0; 2295556cfe8SVlastimil Babka page_ext = page_ext_next(page_ext); 2307e2f2a0cSVlastimil Babka } 231b1d5488aSCharan Teja Kalla page_ext_put(page_ext); 232e2cfc911SJoonsoo Kim } 233e2cfc911SJoonsoo Kim 23419138349SMatthew Wilcox (Oracle) void __folio_copy_owner(struct folio *newfolio, struct folio *old) 235d435edcaSVlastimil Babka { 236b1d5488aSCharan Teja Kalla struct page_ext *old_ext; 237b1d5488aSCharan Teja Kalla struct page_ext *new_ext; 2389300d8dfSJoonsoo Kim struct page_owner *old_page_owner, *new_page_owner; 239d435edcaSVlastimil Babka 240b1d5488aSCharan Teja Kalla old_ext = page_ext_get(&old->page); 241b1d5488aSCharan Teja Kalla if (unlikely(!old_ext)) 242f86e4271SYang Shi return; 243f86e4271SYang Shi 244b1d5488aSCharan Teja Kalla new_ext = page_ext_get(&newfolio->page); 245b1d5488aSCharan Teja Kalla if (unlikely(!new_ext)) { 246b1d5488aSCharan Teja Kalla page_ext_put(old_ext); 247b1d5488aSCharan Teja Kalla return; 248b1d5488aSCharan Teja Kalla } 249b1d5488aSCharan Teja Kalla 2509300d8dfSJoonsoo Kim old_page_owner = get_page_owner(old_ext); 2519300d8dfSJoonsoo Kim new_page_owner = get_page_owner(new_ext); 2529300d8dfSJoonsoo Kim new_page_owner->order = old_page_owner->order; 2539300d8dfSJoonsoo Kim new_page_owner->gfp_mask = old_page_owner->gfp_mask; 2549300d8dfSJoonsoo Kim new_page_owner->last_migrate_reason = 2559300d8dfSJoonsoo Kim old_page_owner->last_migrate_reason; 2569300d8dfSJoonsoo Kim new_page_owner->handle = old_page_owner->handle; 2579cc7e96aSLiam Mark new_page_owner->pid = old_page_owner->pid; 258bf215eabSYixuan Cao new_page_owner->tgid = old_page_owner->tgid; 2591b5c65b6SBarry Song new_page_owner->free_pid = old_page_owner->free_pid; 2601b5c65b6SBarry Song new_page_owner->free_tgid = old_page_owner->free_tgid; 2619cc7e96aSLiam Mark new_page_owner->ts_nsec = old_page_owner->ts_nsec; 262866b4852SGeorgi Djakov new_page_owner->free_ts_nsec = old_page_owner->ts_nsec; 263865ed6a3SWaiman Long strcpy(new_page_owner->comm, old_page_owner->comm); 264d435edcaSVlastimil Babka 265d435edcaSVlastimil Babka /* 26619138349SMatthew Wilcox (Oracle) * We don't clear the bit on the old folio as it's going to be freed 267d435edcaSVlastimil Babka * after migration. Until then, the info can be useful in case of 268f0953a1bSIngo Molnar * a bug, and the overall stats will be off a bit only temporarily. 269d435edcaSVlastimil Babka * Also, migrate_misplaced_transhuge_page() can still fail the 27019138349SMatthew Wilcox (Oracle) * migration and then we want the old folio to retain the info. But 271d435edcaSVlastimil Babka * in that case we also don't need to explicitly clear the info from 272d435edcaSVlastimil Babka * the new page, which will be freed. 273d435edcaSVlastimil Babka */ 274d435edcaSVlastimil Babka __set_bit(PAGE_EXT_OWNER, &new_ext->flags); 275fdf3bf80SVlastimil Babka __set_bit(PAGE_EXT_OWNER_ALLOCATED, &new_ext->flags); 276b1d5488aSCharan Teja Kalla page_ext_put(new_ext); 277b1d5488aSCharan Teja Kalla page_ext_put(old_ext); 278d435edcaSVlastimil Babka } 279d435edcaSVlastimil Babka 280e2f612e6SJoonsoo Kim void pagetypeinfo_showmixedcount_print(struct seq_file *m, 281e2f612e6SJoonsoo Kim pg_data_t *pgdat, struct zone *zone) 282e2f612e6SJoonsoo Kim { 283e2f612e6SJoonsoo Kim struct page *page; 284e2f612e6SJoonsoo Kim struct page_ext *page_ext; 2859300d8dfSJoonsoo Kim struct page_owner *page_owner; 2861d2cae8eSMiaohe Lin unsigned long pfn, block_end_pfn; 2871d2cae8eSMiaohe Lin unsigned long end_pfn = zone_end_pfn(zone); 288e2f612e6SJoonsoo Kim unsigned long count[MIGRATE_TYPES] = { 0, }; 289e2f612e6SJoonsoo Kim int pageblock_mt, page_mt; 290e2f612e6SJoonsoo Kim int i; 291e2f612e6SJoonsoo Kim 292e2f612e6SJoonsoo Kim /* Scan block by block. First and last block may be incomplete */ 293e2f612e6SJoonsoo Kim pfn = zone->zone_start_pfn; 294e2f612e6SJoonsoo Kim 295e2f612e6SJoonsoo Kim /* 296e2f612e6SJoonsoo Kim * Walk the zone in pageblock_nr_pages steps. If a page block spans 297e2f612e6SJoonsoo Kim * a zone boundary, it will be double counted between zones. This does 298e2f612e6SJoonsoo Kim * not matter as the mixed block count will still be correct 299e2f612e6SJoonsoo Kim */ 300e2f612e6SJoonsoo Kim for (; pfn < end_pfn; ) { 301a26ee565SQian Cai page = pfn_to_online_page(pfn); 302a26ee565SQian Cai if (!page) { 303e2f612e6SJoonsoo Kim pfn = ALIGN(pfn + 1, MAX_ORDER_NR_PAGES); 304e2f612e6SJoonsoo Kim continue; 305e2f612e6SJoonsoo Kim } 306e2f612e6SJoonsoo Kim 3074f9bc69aSKefeng Wang block_end_pfn = pageblock_end_pfn(pfn); 308e2f612e6SJoonsoo Kim block_end_pfn = min(block_end_pfn, end_pfn); 309e2f612e6SJoonsoo Kim 310e2f612e6SJoonsoo Kim pageblock_mt = get_pageblock_migratetype(page); 311e2f612e6SJoonsoo Kim 312e2f612e6SJoonsoo Kim for (; pfn < block_end_pfn; pfn++) { 313a26ee565SQian Cai /* The pageblock is online, no need to recheck. */ 314e2f612e6SJoonsoo Kim page = pfn_to_page(pfn); 315e2f612e6SJoonsoo Kim 316e2f612e6SJoonsoo Kim if (page_zone(page) != zone) 317e2f612e6SJoonsoo Kim continue; 318e2f612e6SJoonsoo Kim 319e2f612e6SJoonsoo Kim if (PageBuddy(page)) { 320727c080fSVinayak Menon unsigned long freepage_order; 321727c080fSVinayak Menon 322ab130f91SMatthew Wilcox (Oracle) freepage_order = buddy_order_unsafe(page); 323*5e0a760bSKirill A. Shutemov if (freepage_order <= MAX_PAGE_ORDER) 324727c080fSVinayak Menon pfn += (1UL << freepage_order) - 1; 325e2f612e6SJoonsoo Kim continue; 326e2f612e6SJoonsoo Kim } 327e2f612e6SJoonsoo Kim 328e2f612e6SJoonsoo Kim if (PageReserved(page)) 329e2f612e6SJoonsoo Kim continue; 330e2f612e6SJoonsoo Kim 331b1d5488aSCharan Teja Kalla page_ext = page_ext_get(page); 332e2f612e6SJoonsoo Kim if (unlikely(!page_ext)) 333e2f612e6SJoonsoo Kim continue; 334e2f612e6SJoonsoo Kim 335fdf3bf80SVlastimil Babka if (!test_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags)) 336b1d5488aSCharan Teja Kalla goto ext_put_continue; 337e2f612e6SJoonsoo Kim 3389300d8dfSJoonsoo Kim page_owner = get_page_owner(page_ext); 33901c0bfe0SWei Yang page_mt = gfp_migratetype(page_owner->gfp_mask); 340e2f612e6SJoonsoo Kim if (pageblock_mt != page_mt) { 341e2f612e6SJoonsoo Kim if (is_migrate_cma(pageblock_mt)) 342e2f612e6SJoonsoo Kim count[MIGRATE_MOVABLE]++; 343e2f612e6SJoonsoo Kim else 344e2f612e6SJoonsoo Kim count[pageblock_mt]++; 345e2f612e6SJoonsoo Kim 346e2f612e6SJoonsoo Kim pfn = block_end_pfn; 347b1d5488aSCharan Teja Kalla page_ext_put(page_ext); 348e2f612e6SJoonsoo Kim break; 349e2f612e6SJoonsoo Kim } 3509300d8dfSJoonsoo Kim pfn += (1UL << page_owner->order) - 1; 351b1d5488aSCharan Teja Kalla ext_put_continue: 352b1d5488aSCharan Teja Kalla page_ext_put(page_ext); 353e2f612e6SJoonsoo Kim } 354e2f612e6SJoonsoo Kim } 355e2f612e6SJoonsoo Kim 356e2f612e6SJoonsoo Kim /* Print counts */ 357e2f612e6SJoonsoo Kim seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name); 358e2f612e6SJoonsoo Kim for (i = 0; i < MIGRATE_TYPES; i++) 359e2f612e6SJoonsoo Kim seq_printf(m, "%12lu ", count[i]); 360e2f612e6SJoonsoo Kim seq_putc(m, '\n'); 361e2f612e6SJoonsoo Kim } 362e2f612e6SJoonsoo Kim 363fcf89358SWaiman Long /* 364fcf89358SWaiman Long * Looking for memcg information and print it out 365fcf89358SWaiman Long */ 366fcf89358SWaiman Long static inline int print_page_owner_memcg(char *kbuf, size_t count, int ret, 367fcf89358SWaiman Long struct page *page) 368fcf89358SWaiman Long { 369fcf89358SWaiman Long #ifdef CONFIG_MEMCG 370fcf89358SWaiman Long unsigned long memcg_data; 371fcf89358SWaiman Long struct mem_cgroup *memcg; 372fcf89358SWaiman Long bool online; 373fcf89358SWaiman Long char name[80]; 374fcf89358SWaiman Long 375fcf89358SWaiman Long rcu_read_lock(); 376fcf89358SWaiman Long memcg_data = READ_ONCE(page->memcg_data); 377fcf89358SWaiman Long if (!memcg_data) 378fcf89358SWaiman Long goto out_unlock; 379fcf89358SWaiman Long 380fcf89358SWaiman Long if (memcg_data & MEMCG_DATA_OBJCGS) 381fcf89358SWaiman Long ret += scnprintf(kbuf + ret, count - ret, 382fcf89358SWaiman Long "Slab cache page\n"); 383fcf89358SWaiman Long 384fcf89358SWaiman Long memcg = page_memcg_check(page); 385fcf89358SWaiman Long if (!memcg) 386fcf89358SWaiman Long goto out_unlock; 387fcf89358SWaiman Long 388fcf89358SWaiman Long online = (memcg->css.flags & CSS_ONLINE); 389fcf89358SWaiman Long cgroup_name(memcg->css.cgroup, name, sizeof(name)); 390fcf89358SWaiman Long ret += scnprintf(kbuf + ret, count - ret, 391fcf89358SWaiman Long "Charged %sto %smemcg %s\n", 392fcf89358SWaiman Long PageMemcgKmem(page) ? "(via objcg) " : "", 393fcf89358SWaiman Long online ? "" : "offline ", 394fcf89358SWaiman Long name); 395fcf89358SWaiman Long out_unlock: 396fcf89358SWaiman Long rcu_read_unlock(); 397fcf89358SWaiman Long #endif /* CONFIG_MEMCG */ 398fcf89358SWaiman Long 399fcf89358SWaiman Long return ret; 400fcf89358SWaiman Long } 401fcf89358SWaiman Long 40248c96a36SJoonsoo Kim static ssize_t 40348c96a36SJoonsoo Kim print_page_owner(char __user *buf, size_t count, unsigned long pfn, 4049300d8dfSJoonsoo Kim struct page *page, struct page_owner *page_owner, 405f2ca0b55SJoonsoo Kim depot_stack_handle_t handle) 40648c96a36SJoonsoo Kim { 407af52bf6bSThomas Gleixner int ret, pageblock_mt, page_mt; 40848c96a36SJoonsoo Kim char *kbuf; 40948c96a36SJoonsoo Kim 410c8f61cfcSMiles Chen count = min_t(size_t, count, PAGE_SIZE); 41148c96a36SJoonsoo Kim kbuf = kmalloc(count, GFP_KERNEL); 41248c96a36SJoonsoo Kim if (!kbuf) 41348c96a36SJoonsoo Kim return -ENOMEM; 41448c96a36SJoonsoo Kim 4153ebc4397SWaiman Long ret = scnprintf(kbuf, count, 416b459f090SAudra Mitchell "Page allocated via order %u, mask %#x(%pGg), pid %d, tgid %d (%s), ts %llu ns\n", 4179300d8dfSJoonsoo Kim page_owner->order, page_owner->gfp_mask, 4189cc7e96aSLiam Mark &page_owner->gfp_mask, page_owner->pid, 419bf215eabSYixuan Cao page_owner->tgid, page_owner->comm, 420b459f090SAudra Mitchell page_owner->ts_nsec); 42148c96a36SJoonsoo Kim 42248c96a36SJoonsoo Kim /* Print information relevant to grouping pages by mobility */ 4230b423ca2SMel Gorman pageblock_mt = get_pageblock_migratetype(page); 42401c0bfe0SWei Yang page_mt = gfp_migratetype(page_owner->gfp_mask); 4253ebc4397SWaiman Long ret += scnprintf(kbuf + ret, count - ret, 426399fd496SKassey Li "PFN 0x%lx type %s Block %lu type %s Flags %pGp\n", 42748c96a36SJoonsoo Kim pfn, 42860f30350SVlastimil Babka migratetype_names[page_mt], 42948c96a36SJoonsoo Kim pfn >> pageblock_order, 43060f30350SVlastimil Babka migratetype_names[pageblock_mt], 43123efd080SMatthew Wilcox (Oracle) &page->flags); 43248c96a36SJoonsoo Kim 4330f68d45eSImran Khan ret += stack_depot_snprint(handle, kbuf + ret, count - ret, 0); 43448c96a36SJoonsoo Kim if (ret >= count) 43548c96a36SJoonsoo Kim goto err; 43648c96a36SJoonsoo Kim 4379300d8dfSJoonsoo Kim if (page_owner->last_migrate_reason != -1) { 4383ebc4397SWaiman Long ret += scnprintf(kbuf + ret, count - ret, 4397cd12b4aSVlastimil Babka "Page has been migrated, last migrate reason: %s\n", 4409300d8dfSJoonsoo Kim migrate_reason_names[page_owner->last_migrate_reason]); 4417cd12b4aSVlastimil Babka } 4427cd12b4aSVlastimil Babka 443fcf89358SWaiman Long ret = print_page_owner_memcg(kbuf, count, ret, page); 444fcf89358SWaiman Long 44548c96a36SJoonsoo Kim ret += snprintf(kbuf + ret, count - ret, "\n"); 44648c96a36SJoonsoo Kim if (ret >= count) 44748c96a36SJoonsoo Kim goto err; 44848c96a36SJoonsoo Kim 44948c96a36SJoonsoo Kim if (copy_to_user(buf, kbuf, ret)) 45048c96a36SJoonsoo Kim ret = -EFAULT; 45148c96a36SJoonsoo Kim 45248c96a36SJoonsoo Kim kfree(kbuf); 45348c96a36SJoonsoo Kim return ret; 45448c96a36SJoonsoo Kim 45548c96a36SJoonsoo Kim err: 45648c96a36SJoonsoo Kim kfree(kbuf); 45748c96a36SJoonsoo Kim return -ENOMEM; 45848c96a36SJoonsoo Kim } 45948c96a36SJoonsoo Kim 4608bf6f451SMatthew Wilcox (Oracle) void __dump_page_owner(const struct page *page) 4614e462112SVlastimil Babka { 462b1d5488aSCharan Teja Kalla struct page_ext *page_ext = page_ext_get((void *)page); 4639300d8dfSJoonsoo Kim struct page_owner *page_owner; 464f2ca0b55SJoonsoo Kim depot_stack_handle_t handle; 4658285027fSSudip Mukherjee gfp_t gfp_mask; 4668285027fSSudip Mukherjee int mt; 4674e462112SVlastimil Babka 468f86e4271SYang Shi if (unlikely(!page_ext)) { 469f86e4271SYang Shi pr_alert("There is not page extension available.\n"); 470f86e4271SYang Shi return; 471f86e4271SYang Shi } 4729300d8dfSJoonsoo Kim 4739300d8dfSJoonsoo Kim page_owner = get_page_owner(page_ext); 4749300d8dfSJoonsoo Kim gfp_mask = page_owner->gfp_mask; 47501c0bfe0SWei Yang mt = gfp_migratetype(gfp_mask); 476f86e4271SYang Shi 4774e462112SVlastimil Babka if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags)) { 47837389167SVlastimil Babka pr_alert("page_owner info is not present (never set?)\n"); 479b1d5488aSCharan Teja Kalla page_ext_put(page_ext); 4804e462112SVlastimil Babka return; 4814e462112SVlastimil Babka } 4824e462112SVlastimil Babka 483fdf3bf80SVlastimil Babka if (test_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags)) 48437389167SVlastimil Babka pr_alert("page_owner tracks the page as allocated\n"); 48537389167SVlastimil Babka else 48637389167SVlastimil Babka pr_alert("page_owner tracks the page as freed\n"); 48737389167SVlastimil Babka 488bf215eabSYixuan Cao pr_alert("page last allocated via order %u, migratetype %s, gfp_mask %#x(%pGg), pid %d, tgid %d (%s), ts %llu, free_ts %llu\n", 4899cc7e96aSLiam Mark page_owner->order, migratetype_names[mt], gfp_mask, &gfp_mask, 490bf215eabSYixuan Cao page_owner->pid, page_owner->tgid, page_owner->comm, 491bf215eabSYixuan Cao page_owner->ts_nsec, page_owner->free_ts_nsec); 49237389167SVlastimil Babka 4939300d8dfSJoonsoo Kim handle = READ_ONCE(page_owner->handle); 494505be481SImran Khan if (!handle) 49537389167SVlastimil Babka pr_alert("page_owner allocation stack trace missing\n"); 496505be481SImran Khan else 497505be481SImran Khan stack_depot_print(handle); 4984e462112SVlastimil Babka 4998974558fSVlastimil Babka handle = READ_ONCE(page_owner->free_handle); 5008974558fSVlastimil Babka if (!handle) { 5018974558fSVlastimil Babka pr_alert("page_owner free stack trace missing\n"); 5028974558fSVlastimil Babka } else { 5031b5c65b6SBarry Song pr_alert("page last free pid %d tgid %d stack trace:\n", 5041b5c65b6SBarry Song page_owner->free_pid, page_owner->free_tgid); 505505be481SImran Khan stack_depot_print(handle); 5068974558fSVlastimil Babka } 5078974558fSVlastimil Babka 5089300d8dfSJoonsoo Kim if (page_owner->last_migrate_reason != -1) 5094e462112SVlastimil Babka pr_alert("page has been migrated, last migrate reason: %s\n", 5109300d8dfSJoonsoo Kim migrate_reason_names[page_owner->last_migrate_reason]); 511b1d5488aSCharan Teja Kalla page_ext_put(page_ext); 5124e462112SVlastimil Babka } 5134e462112SVlastimil Babka 51448c96a36SJoonsoo Kim static ssize_t 51548c96a36SJoonsoo Kim read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos) 51648c96a36SJoonsoo Kim { 51748c96a36SJoonsoo Kim unsigned long pfn; 51848c96a36SJoonsoo Kim struct page *page; 51948c96a36SJoonsoo Kim struct page_ext *page_ext; 5209300d8dfSJoonsoo Kim struct page_owner *page_owner; 521f2ca0b55SJoonsoo Kim depot_stack_handle_t handle; 52248c96a36SJoonsoo Kim 5237dd80b8aSVlastimil Babka if (!static_branch_unlikely(&page_owner_inited)) 52448c96a36SJoonsoo Kim return -EINVAL; 52548c96a36SJoonsoo Kim 52648c96a36SJoonsoo Kim page = NULL; 5278f0efa81SKassey Li if (*ppos == 0) 5288f0efa81SKassey Li pfn = min_low_pfn; 5298f0efa81SKassey Li else 5308f0efa81SKassey Li pfn = *ppos; 53148c96a36SJoonsoo Kim /* Find a valid PFN or the start of a MAX_ORDER_NR_PAGES area */ 53248c96a36SJoonsoo Kim while (!pfn_valid(pfn) && (pfn & (MAX_ORDER_NR_PAGES - 1)) != 0) 53348c96a36SJoonsoo Kim pfn++; 53448c96a36SJoonsoo Kim 53548c96a36SJoonsoo Kim /* Find an allocated page */ 53648c96a36SJoonsoo Kim for (; pfn < max_pfn; pfn++) { 53748c96a36SJoonsoo Kim /* 538b1d5488aSCharan Teja Kalla * This temporary page_owner is required so 539b1d5488aSCharan Teja Kalla * that we can avoid the context switches while holding 540b1d5488aSCharan Teja Kalla * the rcu lock and copying the page owner information to 541b1d5488aSCharan Teja Kalla * user through copy_to_user() or GFP_KERNEL allocations. 542b1d5488aSCharan Teja Kalla */ 543b1d5488aSCharan Teja Kalla struct page_owner page_owner_tmp; 544b1d5488aSCharan Teja Kalla 545b1d5488aSCharan Teja Kalla /* 54648c96a36SJoonsoo Kim * If the new page is in a new MAX_ORDER_NR_PAGES area, 54748c96a36SJoonsoo Kim * validate the area as existing, skip it if not 54848c96a36SJoonsoo Kim */ 54948c96a36SJoonsoo Kim if ((pfn & (MAX_ORDER_NR_PAGES - 1)) == 0 && !pfn_valid(pfn)) { 55048c96a36SJoonsoo Kim pfn += MAX_ORDER_NR_PAGES - 1; 55148c96a36SJoonsoo Kim continue; 55248c96a36SJoonsoo Kim } 55348c96a36SJoonsoo Kim 55448c96a36SJoonsoo Kim page = pfn_to_page(pfn); 55548c96a36SJoonsoo Kim if (PageBuddy(page)) { 556ab130f91SMatthew Wilcox (Oracle) unsigned long freepage_order = buddy_order_unsafe(page); 55748c96a36SJoonsoo Kim 558*5e0a760bSKirill A. Shutemov if (freepage_order <= MAX_PAGE_ORDER) 55948c96a36SJoonsoo Kim pfn += (1UL << freepage_order) - 1; 56048c96a36SJoonsoo Kim continue; 56148c96a36SJoonsoo Kim } 56248c96a36SJoonsoo Kim 563b1d5488aSCharan Teja Kalla page_ext = page_ext_get(page); 564f86e4271SYang Shi if (unlikely(!page_ext)) 565f86e4271SYang Shi continue; 56648c96a36SJoonsoo Kim 56748c96a36SJoonsoo Kim /* 56861cf5febSJoonsoo Kim * Some pages could be missed by concurrent allocation or free, 56961cf5febSJoonsoo Kim * because we don't hold the zone lock. 57048c96a36SJoonsoo Kim */ 57148c96a36SJoonsoo Kim if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags)) 572b1d5488aSCharan Teja Kalla goto ext_put_continue; 57348c96a36SJoonsoo Kim 57437389167SVlastimil Babka /* 57537389167SVlastimil Babka * Although we do have the info about past allocation of free 57637389167SVlastimil Babka * pages, it's not relevant for current memory usage. 57737389167SVlastimil Babka */ 578fdf3bf80SVlastimil Babka if (!test_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags)) 579b1d5488aSCharan Teja Kalla goto ext_put_continue; 58037389167SVlastimil Babka 5819300d8dfSJoonsoo Kim page_owner = get_page_owner(page_ext); 5829300d8dfSJoonsoo Kim 583f2ca0b55SJoonsoo Kim /* 5847e2f2a0cSVlastimil Babka * Don't print "tail" pages of high-order allocations as that 5857e2f2a0cSVlastimil Babka * would inflate the stats. 5867e2f2a0cSVlastimil Babka */ 5877e2f2a0cSVlastimil Babka if (!IS_ALIGNED(pfn, 1 << page_owner->order)) 588b1d5488aSCharan Teja Kalla goto ext_put_continue; 5897e2f2a0cSVlastimil Babka 5907e2f2a0cSVlastimil Babka /* 591f2ca0b55SJoonsoo Kim * Access to page_ext->handle isn't synchronous so we should 592f2ca0b55SJoonsoo Kim * be careful to access it. 593f2ca0b55SJoonsoo Kim */ 5949300d8dfSJoonsoo Kim handle = READ_ONCE(page_owner->handle); 595f2ca0b55SJoonsoo Kim if (!handle) 596b1d5488aSCharan Teja Kalla goto ext_put_continue; 597f2ca0b55SJoonsoo Kim 59848c96a36SJoonsoo Kim /* Record the next PFN to read in the file offset */ 5998f0efa81SKassey Li *ppos = pfn + 1; 60048c96a36SJoonsoo Kim 601b1d5488aSCharan Teja Kalla page_owner_tmp = *page_owner; 602b1d5488aSCharan Teja Kalla page_ext_put(page_ext); 603f2ca0b55SJoonsoo Kim return print_page_owner(buf, count, pfn, page, 604b1d5488aSCharan Teja Kalla &page_owner_tmp, handle); 605b1d5488aSCharan Teja Kalla ext_put_continue: 606b1d5488aSCharan Teja Kalla page_ext_put(page_ext); 60748c96a36SJoonsoo Kim } 60848c96a36SJoonsoo Kim 60948c96a36SJoonsoo Kim return 0; 61048c96a36SJoonsoo Kim } 61148c96a36SJoonsoo Kim 6128f0efa81SKassey Li static loff_t lseek_page_owner(struct file *file, loff_t offset, int orig) 6138f0efa81SKassey Li { 6148f0efa81SKassey Li switch (orig) { 6158f0efa81SKassey Li case SEEK_SET: 6168f0efa81SKassey Li file->f_pos = offset; 6178f0efa81SKassey Li break; 6188f0efa81SKassey Li case SEEK_CUR: 6198f0efa81SKassey Li file->f_pos += offset; 6208f0efa81SKassey Li break; 6218f0efa81SKassey Li default: 6228f0efa81SKassey Li return -EINVAL; 6238f0efa81SKassey Li } 6248f0efa81SKassey Li return file->f_pos; 6258f0efa81SKassey Li } 6268f0efa81SKassey Li 62761cf5febSJoonsoo Kim static void init_pages_in_zone(pg_data_t *pgdat, struct zone *zone) 62861cf5febSJoonsoo Kim { 6296787c1daSOscar Salvador unsigned long pfn = zone->zone_start_pfn; 6306787c1daSOscar Salvador unsigned long end_pfn = zone_end_pfn(zone); 63161cf5febSJoonsoo Kim unsigned long count = 0; 63261cf5febSJoonsoo Kim 63361cf5febSJoonsoo Kim /* 63461cf5febSJoonsoo Kim * Walk the zone in pageblock_nr_pages steps. If a page block spans 63561cf5febSJoonsoo Kim * a zone boundary, it will be double counted between zones. This does 63661cf5febSJoonsoo Kim * not matter as the mixed block count will still be correct 63761cf5febSJoonsoo Kim */ 63861cf5febSJoonsoo Kim for (; pfn < end_pfn; ) { 6396787c1daSOscar Salvador unsigned long block_end_pfn; 6406787c1daSOscar Salvador 64161cf5febSJoonsoo Kim if (!pfn_valid(pfn)) { 64261cf5febSJoonsoo Kim pfn = ALIGN(pfn + 1, MAX_ORDER_NR_PAGES); 64361cf5febSJoonsoo Kim continue; 64461cf5febSJoonsoo Kim } 64561cf5febSJoonsoo Kim 6464f9bc69aSKefeng Wang block_end_pfn = pageblock_end_pfn(pfn); 64761cf5febSJoonsoo Kim block_end_pfn = min(block_end_pfn, end_pfn); 64861cf5febSJoonsoo Kim 64961cf5febSJoonsoo Kim for (; pfn < block_end_pfn; pfn++) { 650859a85ddSMike Rapoport struct page *page = pfn_to_page(pfn); 6516787c1daSOscar Salvador struct page_ext *page_ext; 6526787c1daSOscar Salvador 6539d43f5aeSJoonsoo Kim if (page_zone(page) != zone) 6549d43f5aeSJoonsoo Kim continue; 6559d43f5aeSJoonsoo Kim 65661cf5febSJoonsoo Kim /* 65710903027SVlastimil Babka * To avoid having to grab zone->lock, be a little 65810903027SVlastimil Babka * careful when reading buddy page order. The only 65910903027SVlastimil Babka * danger is that we skip too much and potentially miss 66010903027SVlastimil Babka * some early allocated pages, which is better than 66110903027SVlastimil Babka * heavy lock contention. 66261cf5febSJoonsoo Kim */ 66361cf5febSJoonsoo Kim if (PageBuddy(page)) { 664ab130f91SMatthew Wilcox (Oracle) unsigned long order = buddy_order_unsafe(page); 66510903027SVlastimil Babka 666*5e0a760bSKirill A. Shutemov if (order > 0 && order <= MAX_PAGE_ORDER) 66710903027SVlastimil Babka pfn += (1UL << order) - 1; 66861cf5febSJoonsoo Kim continue; 66961cf5febSJoonsoo Kim } 67061cf5febSJoonsoo Kim 67161cf5febSJoonsoo Kim if (PageReserved(page)) 67261cf5febSJoonsoo Kim continue; 67361cf5febSJoonsoo Kim 674b1d5488aSCharan Teja Kalla page_ext = page_ext_get(page); 675f86e4271SYang Shi if (unlikely(!page_ext)) 676f86e4271SYang Shi continue; 67761cf5febSJoonsoo Kim 678dab4ead1SVlastimil Babka /* Maybe overlapping zone */ 67961cf5febSJoonsoo Kim if (test_bit(PAGE_EXT_OWNER, &page_ext->flags)) 680b1d5488aSCharan Teja Kalla goto ext_put_continue; 68161cf5febSJoonsoo Kim 68261cf5febSJoonsoo Kim /* Found early allocated page */ 68364ea78d2Szhongjiang-ali __set_page_owner_handle(page_ext, early_handle, 6847e2f2a0cSVlastimil Babka 0, 0); 68561cf5febSJoonsoo Kim count++; 686b1d5488aSCharan Teja Kalla ext_put_continue: 687b1d5488aSCharan Teja Kalla page_ext_put(page_ext); 68861cf5febSJoonsoo Kim } 68910903027SVlastimil Babka cond_resched(); 69061cf5febSJoonsoo Kim } 69161cf5febSJoonsoo Kim 69261cf5febSJoonsoo Kim pr_info("Node %d, zone %8s: page owner found early allocated %lu pages\n", 69361cf5febSJoonsoo Kim pgdat->node_id, zone->name, count); 69461cf5febSJoonsoo Kim } 69561cf5febSJoonsoo Kim 69661cf5febSJoonsoo Kim static void init_zones_in_node(pg_data_t *pgdat) 69761cf5febSJoonsoo Kim { 69861cf5febSJoonsoo Kim struct zone *zone; 69961cf5febSJoonsoo Kim struct zone *node_zones = pgdat->node_zones; 70061cf5febSJoonsoo Kim 70161cf5febSJoonsoo Kim for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) { 70261cf5febSJoonsoo Kim if (!populated_zone(zone)) 70361cf5febSJoonsoo Kim continue; 70461cf5febSJoonsoo Kim 70561cf5febSJoonsoo Kim init_pages_in_zone(pgdat, zone); 70661cf5febSJoonsoo Kim } 70761cf5febSJoonsoo Kim } 70861cf5febSJoonsoo Kim 70961cf5febSJoonsoo Kim static void init_early_allocated_pages(void) 71061cf5febSJoonsoo Kim { 71161cf5febSJoonsoo Kim pg_data_t *pgdat; 71261cf5febSJoonsoo Kim 71361cf5febSJoonsoo Kim for_each_online_pgdat(pgdat) 71461cf5febSJoonsoo Kim init_zones_in_node(pgdat); 71561cf5febSJoonsoo Kim } 71661cf5febSJoonsoo Kim 71748c96a36SJoonsoo Kim static const struct file_operations proc_page_owner_operations = { 71848c96a36SJoonsoo Kim .read = read_page_owner, 7198f0efa81SKassey Li .llseek = lseek_page_owner, 72048c96a36SJoonsoo Kim }; 72148c96a36SJoonsoo Kim 72248c96a36SJoonsoo Kim static int __init pageowner_init(void) 72348c96a36SJoonsoo Kim { 7247dd80b8aSVlastimil Babka if (!static_branch_unlikely(&page_owner_inited)) { 72548c96a36SJoonsoo Kim pr_info("page_owner is disabled\n"); 72648c96a36SJoonsoo Kim return 0; 72748c96a36SJoonsoo Kim } 72848c96a36SJoonsoo Kim 729d9f7979cSGreg Kroah-Hartman debugfs_create_file("page_owner", 0400, NULL, NULL, 730d9f7979cSGreg Kroah-Hartman &proc_page_owner_operations); 73148c96a36SJoonsoo Kim 732d9f7979cSGreg Kroah-Hartman return 0; 73348c96a36SJoonsoo Kim } 73444c5af96SPaul Gortmaker late_initcall(pageowner_init) 735