1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 2eefa864bSJoonsoo Kim #include <linux/mm.h> 3eefa864bSJoonsoo Kim #include <linux/mmzone.h> 457c8a661SMike Rapoport #include <linux/memblock.h> 5eefa864bSJoonsoo Kim #include <linux/page_ext.h> 6eefa864bSJoonsoo Kim #include <linux/memory.h> 7eefa864bSJoonsoo Kim #include <linux/vmalloc.h> 8eefa864bSJoonsoo Kim #include <linux/kmemleak.h> 948c96a36SJoonsoo Kim #include <linux/page_owner.h> 1033c3fc71SVladimir Davydov #include <linux/page_idle.h> 11eefa864bSJoonsoo Kim 12eefa864bSJoonsoo Kim /* 13eefa864bSJoonsoo Kim * struct page extension 14eefa864bSJoonsoo Kim * 15eefa864bSJoonsoo Kim * This is the feature to manage memory for extended data per page. 16eefa864bSJoonsoo Kim * 17eefa864bSJoonsoo Kim * Until now, we must modify struct page itself to store extra data per page. 18eefa864bSJoonsoo Kim * This requires rebuilding the kernel and it is really time consuming process. 19eefa864bSJoonsoo Kim * And, sometimes, rebuild is impossible due to third party module dependency. 20eefa864bSJoonsoo Kim * At last, enlarging struct page could cause un-wanted system behaviour change. 21eefa864bSJoonsoo Kim * 22eefa864bSJoonsoo Kim * This feature is intended to overcome above mentioned problems. This feature 23eefa864bSJoonsoo Kim * allocates memory for extended data per page in certain place rather than 24eefa864bSJoonsoo Kim * the struct page itself. This memory can be accessed by the accessor 25eefa864bSJoonsoo Kim * functions provided by this code. During the boot process, it checks whether 26eefa864bSJoonsoo Kim * allocation of huge chunk of memory is needed or not. If not, it avoids 27eefa864bSJoonsoo Kim * allocating memory at all. With this advantage, we can include this feature 28eefa864bSJoonsoo Kim * into the kernel in default and can avoid rebuild and solve related problems. 29eefa864bSJoonsoo Kim * 30eefa864bSJoonsoo Kim * To help these things to work well, there are two callbacks for clients. One 31eefa864bSJoonsoo Kim * is the need callback which is mandatory if user wants to avoid useless 32eefa864bSJoonsoo Kim * memory allocation at boot-time. The other is optional, init callback, which 33eefa864bSJoonsoo Kim * is used to do proper initialization after memory is allocated. 34eefa864bSJoonsoo Kim * 35eefa864bSJoonsoo Kim * The need callback is used to decide whether extended memory allocation is 36eefa864bSJoonsoo Kim * needed or not. Sometimes users want to deactivate some features in this 378958b249SHaitao Shi * boot and extra memory would be unnecessary. In this case, to avoid 38eefa864bSJoonsoo Kim * allocating huge chunk of memory, each clients represent their need of 39eefa864bSJoonsoo Kim * extra memory through the need callback. If one of the need callbacks 40eefa864bSJoonsoo Kim * returns true, it means that someone needs extra memory so that 41eefa864bSJoonsoo Kim * page extension core should allocates memory for page extension. If 42eefa864bSJoonsoo Kim * none of need callbacks return true, memory isn't needed at all in this boot 43eefa864bSJoonsoo Kim * and page extension core can skip to allocate memory. As result, 44eefa864bSJoonsoo Kim * none of memory is wasted. 45eefa864bSJoonsoo Kim * 46980ac167SJoonsoo Kim * When need callback returns true, page_ext checks if there is a request for 47980ac167SJoonsoo Kim * extra memory through size in struct page_ext_operations. If it is non-zero, 48980ac167SJoonsoo Kim * extra space is allocated for each page_ext entry and offset is returned to 49980ac167SJoonsoo Kim * user through offset in struct page_ext_operations. 50980ac167SJoonsoo Kim * 51eefa864bSJoonsoo Kim * The init callback is used to do proper initialization after page extension 52eefa864bSJoonsoo Kim * is completely initialized. In sparse memory system, extra memory is 53eefa864bSJoonsoo Kim * allocated some time later than memmap is allocated. In other words, lifetime 54eefa864bSJoonsoo Kim * of memory for page extension isn't same with memmap for struct page. 55eefa864bSJoonsoo Kim * Therefore, clients can't store extra data until page extension is 56eefa864bSJoonsoo Kim * initialized, even if pages are allocated and used freely. This could 57eefa864bSJoonsoo Kim * cause inadequate state of extra data per page, so, to prevent it, client 58eefa864bSJoonsoo Kim * can utilize this callback to initialize the state of it correctly. 59eefa864bSJoonsoo Kim */ 60eefa864bSJoonsoo Kim 61*1c676e0dSSeongJae Park #if defined(CONFIG_PAGE_IDLE_FLAG) && !defined(CONFIG_64BIT) 62*1c676e0dSSeongJae Park static bool need_page_idle(void) 63*1c676e0dSSeongJae Park { 64*1c676e0dSSeongJae Park return true; 65*1c676e0dSSeongJae Park } 66*1c676e0dSSeongJae Park struct page_ext_operations page_idle_ops = { 67*1c676e0dSSeongJae Park .need = need_page_idle, 68*1c676e0dSSeongJae Park }; 69*1c676e0dSSeongJae Park #endif 70*1c676e0dSSeongJae Park 71eefa864bSJoonsoo Kim static struct page_ext_operations *page_ext_ops[] = { 7248c96a36SJoonsoo Kim #ifdef CONFIG_PAGE_OWNER 7348c96a36SJoonsoo Kim &page_owner_ops, 7448c96a36SJoonsoo Kim #endif 75*1c676e0dSSeongJae Park #if defined(CONFIG_PAGE_IDLE_FLAG) && !defined(CONFIG_64BIT) 7633c3fc71SVladimir Davydov &page_idle_ops, 7733c3fc71SVladimir Davydov #endif 78eefa864bSJoonsoo Kim }; 79eefa864bSJoonsoo Kim 805556cfe8SVlastimil Babka unsigned long page_ext_size = sizeof(struct page_ext); 815556cfe8SVlastimil Babka 82eefa864bSJoonsoo Kim static unsigned long total_usage; 83eefa864bSJoonsoo Kim 84eefa864bSJoonsoo Kim static bool __init invoke_need_callbacks(void) 85eefa864bSJoonsoo Kim { 86eefa864bSJoonsoo Kim int i; 87eefa864bSJoonsoo Kim int entries = ARRAY_SIZE(page_ext_ops); 88980ac167SJoonsoo Kim bool need = false; 89eefa864bSJoonsoo Kim 90eefa864bSJoonsoo Kim for (i = 0; i < entries; i++) { 91980ac167SJoonsoo Kim if (page_ext_ops[i]->need && page_ext_ops[i]->need()) { 925556cfe8SVlastimil Babka page_ext_ops[i]->offset = page_ext_size; 935556cfe8SVlastimil Babka page_ext_size += page_ext_ops[i]->size; 94980ac167SJoonsoo Kim need = true; 95980ac167SJoonsoo Kim } 96eefa864bSJoonsoo Kim } 97eefa864bSJoonsoo Kim 98980ac167SJoonsoo Kim return need; 99eefa864bSJoonsoo Kim } 100eefa864bSJoonsoo Kim 101eefa864bSJoonsoo Kim static void __init invoke_init_callbacks(void) 102eefa864bSJoonsoo Kim { 103eefa864bSJoonsoo Kim int i; 104eefa864bSJoonsoo Kim int entries = ARRAY_SIZE(page_ext_ops); 105eefa864bSJoonsoo Kim 106eefa864bSJoonsoo Kim for (i = 0; i < entries; i++) { 107eefa864bSJoonsoo Kim if (page_ext_ops[i]->init) 108eefa864bSJoonsoo Kim page_ext_ops[i]->init(); 109eefa864bSJoonsoo Kim } 110eefa864bSJoonsoo Kim } 111eefa864bSJoonsoo Kim 1127fb7ab6dSZhenhua Huang #ifndef CONFIG_SPARSEMEM 1137fb7ab6dSZhenhua Huang void __init page_ext_init_flatmem_late(void) 1147fb7ab6dSZhenhua Huang { 1157fb7ab6dSZhenhua Huang invoke_init_callbacks(); 1167fb7ab6dSZhenhua Huang } 1177fb7ab6dSZhenhua Huang #endif 1187fb7ab6dSZhenhua Huang 119980ac167SJoonsoo Kim static inline struct page_ext *get_entry(void *base, unsigned long index) 120980ac167SJoonsoo Kim { 1215556cfe8SVlastimil Babka return base + page_ext_size * index; 122980ac167SJoonsoo Kim } 123980ac167SJoonsoo Kim 1247fb7ab6dSZhenhua Huang #ifndef CONFIG_SPARSEMEM 125eefa864bSJoonsoo Kim 126eefa864bSJoonsoo Kim 127eefa864bSJoonsoo Kim void __meminit pgdat_page_ext_init(struct pglist_data *pgdat) 128eefa864bSJoonsoo Kim { 129eefa864bSJoonsoo Kim pgdat->node_page_ext = NULL; 130eefa864bSJoonsoo Kim } 131eefa864bSJoonsoo Kim 13210ed6341SKirill A. Shutemov struct page_ext *lookup_page_ext(const struct page *page) 133eefa864bSJoonsoo Kim { 134eefa864bSJoonsoo Kim unsigned long pfn = page_to_pfn(page); 1350b06bb3fSJoonsoo Kim unsigned long index; 136eefa864bSJoonsoo Kim struct page_ext *base; 137eefa864bSJoonsoo Kim 138eefa864bSJoonsoo Kim base = NODE_DATA(page_to_nid(page))->node_page_ext; 139eefa864bSJoonsoo Kim /* 140eefa864bSJoonsoo Kim * The sanity checks the page allocator does upon freeing a 141eefa864bSJoonsoo Kim * page can reach here before the page_ext arrays are 142eefa864bSJoonsoo Kim * allocated when feeding a range of pages to the allocator 143eefa864bSJoonsoo Kim * for the first time during bootup or memory hotplug. 144eefa864bSJoonsoo Kim */ 145eefa864bSJoonsoo Kim if (unlikely(!base)) 146eefa864bSJoonsoo Kim return NULL; 1470b06bb3fSJoonsoo Kim index = pfn - round_down(node_start_pfn(page_to_nid(page)), 148eefa864bSJoonsoo Kim MAX_ORDER_NR_PAGES); 149980ac167SJoonsoo Kim return get_entry(base, index); 150eefa864bSJoonsoo Kim } 151eefa864bSJoonsoo Kim 152eefa864bSJoonsoo Kim static int __init alloc_node_page_ext(int nid) 153eefa864bSJoonsoo Kim { 154eefa864bSJoonsoo Kim struct page_ext *base; 155eefa864bSJoonsoo Kim unsigned long table_size; 156eefa864bSJoonsoo Kim unsigned long nr_pages; 157eefa864bSJoonsoo Kim 158eefa864bSJoonsoo Kim nr_pages = NODE_DATA(nid)->node_spanned_pages; 159eefa864bSJoonsoo Kim if (!nr_pages) 160eefa864bSJoonsoo Kim return 0; 161eefa864bSJoonsoo Kim 162eefa864bSJoonsoo Kim /* 163eefa864bSJoonsoo Kim * Need extra space if node range is not aligned with 164eefa864bSJoonsoo Kim * MAX_ORDER_NR_PAGES. When page allocator's buddy algorithm 165eefa864bSJoonsoo Kim * checks buddy's status, range could be out of exact node range. 166eefa864bSJoonsoo Kim */ 167eefa864bSJoonsoo Kim if (!IS_ALIGNED(node_start_pfn(nid), MAX_ORDER_NR_PAGES) || 168eefa864bSJoonsoo Kim !IS_ALIGNED(node_end_pfn(nid), MAX_ORDER_NR_PAGES)) 169eefa864bSJoonsoo Kim nr_pages += MAX_ORDER_NR_PAGES; 170eefa864bSJoonsoo Kim 1715556cfe8SVlastimil Babka table_size = page_ext_size * nr_pages; 172eefa864bSJoonsoo Kim 17326fb3daeSMike Rapoport base = memblock_alloc_try_nid( 174eefa864bSJoonsoo Kim table_size, PAGE_SIZE, __pa(MAX_DMA_ADDRESS), 17597ad1087SMike Rapoport MEMBLOCK_ALLOC_ACCESSIBLE, nid); 176eefa864bSJoonsoo Kim if (!base) 177eefa864bSJoonsoo Kim return -ENOMEM; 178eefa864bSJoonsoo Kim NODE_DATA(nid)->node_page_ext = base; 179eefa864bSJoonsoo Kim total_usage += table_size; 180eefa864bSJoonsoo Kim return 0; 181eefa864bSJoonsoo Kim } 182eefa864bSJoonsoo Kim 183eefa864bSJoonsoo Kim void __init page_ext_init_flatmem(void) 184eefa864bSJoonsoo Kim { 185eefa864bSJoonsoo Kim 186eefa864bSJoonsoo Kim int nid, fail; 187eefa864bSJoonsoo Kim 188eefa864bSJoonsoo Kim if (!invoke_need_callbacks()) 189eefa864bSJoonsoo Kim return; 190eefa864bSJoonsoo Kim 191eefa864bSJoonsoo Kim for_each_online_node(nid) { 192eefa864bSJoonsoo Kim fail = alloc_node_page_ext(nid); 193eefa864bSJoonsoo Kim if (fail) 194eefa864bSJoonsoo Kim goto fail; 195eefa864bSJoonsoo Kim } 196eefa864bSJoonsoo Kim pr_info("allocated %ld bytes of page_ext\n", total_usage); 197eefa864bSJoonsoo Kim return; 198eefa864bSJoonsoo Kim 199eefa864bSJoonsoo Kim fail: 200eefa864bSJoonsoo Kim pr_crit("allocation of page_ext failed.\n"); 201eefa864bSJoonsoo Kim panic("Out of memory"); 202eefa864bSJoonsoo Kim } 203eefa864bSJoonsoo Kim 20443b02ba9SMike Rapoport #else /* CONFIG_FLATMEM */ 205eefa864bSJoonsoo Kim 20610ed6341SKirill A. Shutemov struct page_ext *lookup_page_ext(const struct page *page) 207eefa864bSJoonsoo Kim { 208eefa864bSJoonsoo Kim unsigned long pfn = page_to_pfn(page); 209eefa864bSJoonsoo Kim struct mem_section *section = __pfn_to_section(pfn); 210eefa864bSJoonsoo Kim /* 211eefa864bSJoonsoo Kim * The sanity checks the page allocator does upon freeing a 212eefa864bSJoonsoo Kim * page can reach here before the page_ext arrays are 213eefa864bSJoonsoo Kim * allocated when feeding a range of pages to the allocator 214eefa864bSJoonsoo Kim * for the first time during bootup or memory hotplug. 215eefa864bSJoonsoo Kim */ 216eefa864bSJoonsoo Kim if (!section->page_ext) 217eefa864bSJoonsoo Kim return NULL; 218980ac167SJoonsoo Kim return get_entry(section->page_ext, pfn); 219eefa864bSJoonsoo Kim } 220eefa864bSJoonsoo Kim 221eefa864bSJoonsoo Kim static void *__meminit alloc_page_ext(size_t size, int nid) 222eefa864bSJoonsoo Kim { 223eefa864bSJoonsoo Kim gfp_t flags = GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN; 224eefa864bSJoonsoo Kim void *addr = NULL; 225eefa864bSJoonsoo Kim 226eefa864bSJoonsoo Kim addr = alloc_pages_exact_nid(nid, size, flags); 227eefa864bSJoonsoo Kim if (addr) { 228eefa864bSJoonsoo Kim kmemleak_alloc(addr, size, 1, flags); 229eefa864bSJoonsoo Kim return addr; 230eefa864bSJoonsoo Kim } 231eefa864bSJoonsoo Kim 232eefa864bSJoonsoo Kim addr = vzalloc_node(size, nid); 233eefa864bSJoonsoo Kim 234eefa864bSJoonsoo Kim return addr; 235eefa864bSJoonsoo Kim } 236eefa864bSJoonsoo Kim 237eefa864bSJoonsoo Kim static int __meminit init_section_page_ext(unsigned long pfn, int nid) 238eefa864bSJoonsoo Kim { 239eefa864bSJoonsoo Kim struct mem_section *section; 240eefa864bSJoonsoo Kim struct page_ext *base; 241eefa864bSJoonsoo Kim unsigned long table_size; 242eefa864bSJoonsoo Kim 243eefa864bSJoonsoo Kim section = __pfn_to_section(pfn); 244eefa864bSJoonsoo Kim 245eefa864bSJoonsoo Kim if (section->page_ext) 246eefa864bSJoonsoo Kim return 0; 247eefa864bSJoonsoo Kim 2485556cfe8SVlastimil Babka table_size = page_ext_size * PAGES_PER_SECTION; 249eefa864bSJoonsoo Kim base = alloc_page_ext(table_size, nid); 250eefa864bSJoonsoo Kim 251eefa864bSJoonsoo Kim /* 252eefa864bSJoonsoo Kim * The value stored in section->page_ext is (base - pfn) 253eefa864bSJoonsoo Kim * and it does not point to the memory block allocated above, 254eefa864bSJoonsoo Kim * causing kmemleak false positives. 255eefa864bSJoonsoo Kim */ 256eefa864bSJoonsoo Kim kmemleak_not_leak(base); 257eefa864bSJoonsoo Kim 258eefa864bSJoonsoo Kim if (!base) { 259eefa864bSJoonsoo Kim pr_err("page ext allocation failure\n"); 260eefa864bSJoonsoo Kim return -ENOMEM; 261eefa864bSJoonsoo Kim } 262eefa864bSJoonsoo Kim 263eefa864bSJoonsoo Kim /* 264eefa864bSJoonsoo Kim * The passed "pfn" may not be aligned to SECTION. For the calculation 265eefa864bSJoonsoo Kim * we need to apply a mask. 266eefa864bSJoonsoo Kim */ 267eefa864bSJoonsoo Kim pfn &= PAGE_SECTION_MASK; 2685556cfe8SVlastimil Babka section->page_ext = (void *)base - page_ext_size * pfn; 269eefa864bSJoonsoo Kim total_usage += table_size; 270eefa864bSJoonsoo Kim return 0; 271eefa864bSJoonsoo Kim } 272eefa864bSJoonsoo Kim #ifdef CONFIG_MEMORY_HOTPLUG 273eefa864bSJoonsoo Kim static void free_page_ext(void *addr) 274eefa864bSJoonsoo Kim { 275eefa864bSJoonsoo Kim if (is_vmalloc_addr(addr)) { 276eefa864bSJoonsoo Kim vfree(addr); 277eefa864bSJoonsoo Kim } else { 278eefa864bSJoonsoo Kim struct page *page = virt_to_page(addr); 279eefa864bSJoonsoo Kim size_t table_size; 280eefa864bSJoonsoo Kim 2815556cfe8SVlastimil Babka table_size = page_ext_size * PAGES_PER_SECTION; 282eefa864bSJoonsoo Kim 283eefa864bSJoonsoo Kim BUG_ON(PageReserved(page)); 2840c815854SQian Cai kmemleak_free(addr); 285eefa864bSJoonsoo Kim free_pages_exact(addr, table_size); 286eefa864bSJoonsoo Kim } 287eefa864bSJoonsoo Kim } 288eefa864bSJoonsoo Kim 289eefa864bSJoonsoo Kim static void __free_page_ext(unsigned long pfn) 290eefa864bSJoonsoo Kim { 291eefa864bSJoonsoo Kim struct mem_section *ms; 292eefa864bSJoonsoo Kim struct page_ext *base; 293eefa864bSJoonsoo Kim 294eefa864bSJoonsoo Kim ms = __pfn_to_section(pfn); 295eefa864bSJoonsoo Kim if (!ms || !ms->page_ext) 296eefa864bSJoonsoo Kim return; 297980ac167SJoonsoo Kim base = get_entry(ms->page_ext, pfn); 298eefa864bSJoonsoo Kim free_page_ext(base); 299eefa864bSJoonsoo Kim ms->page_ext = NULL; 300eefa864bSJoonsoo Kim } 301eefa864bSJoonsoo Kim 302eefa864bSJoonsoo Kim static int __meminit online_page_ext(unsigned long start_pfn, 303eefa864bSJoonsoo Kim unsigned long nr_pages, 304eefa864bSJoonsoo Kim int nid) 305eefa864bSJoonsoo Kim { 306eefa864bSJoonsoo Kim unsigned long start, end, pfn; 307eefa864bSJoonsoo Kim int fail = 0; 308eefa864bSJoonsoo Kim 309eefa864bSJoonsoo Kim start = SECTION_ALIGN_DOWN(start_pfn); 310eefa864bSJoonsoo Kim end = SECTION_ALIGN_UP(start_pfn + nr_pages); 311eefa864bSJoonsoo Kim 31298fa15f3SAnshuman Khandual if (nid == NUMA_NO_NODE) { 313eefa864bSJoonsoo Kim /* 314eefa864bSJoonsoo Kim * In this case, "nid" already exists and contains valid memory. 315eefa864bSJoonsoo Kim * "start_pfn" passed to us is a pfn which is an arg for 316eefa864bSJoonsoo Kim * online__pages(), and start_pfn should exist. 317eefa864bSJoonsoo Kim */ 318eefa864bSJoonsoo Kim nid = pfn_to_nid(start_pfn); 319eefa864bSJoonsoo Kim VM_BUG_ON(!node_state(nid, N_ONLINE)); 320eefa864bSJoonsoo Kim } 321eefa864bSJoonsoo Kim 322dccacf8dSDavid Hildenbrand for (pfn = start; !fail && pfn < end; pfn += PAGES_PER_SECTION) 323eefa864bSJoonsoo Kim fail = init_section_page_ext(pfn, nid); 324eefa864bSJoonsoo Kim if (!fail) 325eefa864bSJoonsoo Kim return 0; 326eefa864bSJoonsoo Kim 327eefa864bSJoonsoo Kim /* rollback */ 328eefa864bSJoonsoo Kim for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) 329eefa864bSJoonsoo Kim __free_page_ext(pfn); 330eefa864bSJoonsoo Kim 331eefa864bSJoonsoo Kim return -ENOMEM; 332eefa864bSJoonsoo Kim } 333eefa864bSJoonsoo Kim 334eefa864bSJoonsoo Kim static int __meminit offline_page_ext(unsigned long start_pfn, 335eefa864bSJoonsoo Kim unsigned long nr_pages, int nid) 336eefa864bSJoonsoo Kim { 337eefa864bSJoonsoo Kim unsigned long start, end, pfn; 338eefa864bSJoonsoo Kim 339eefa864bSJoonsoo Kim start = SECTION_ALIGN_DOWN(start_pfn); 340eefa864bSJoonsoo Kim end = SECTION_ALIGN_UP(start_pfn + nr_pages); 341eefa864bSJoonsoo Kim 342eefa864bSJoonsoo Kim for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) 343eefa864bSJoonsoo Kim __free_page_ext(pfn); 344eefa864bSJoonsoo Kim return 0; 345eefa864bSJoonsoo Kim 346eefa864bSJoonsoo Kim } 347eefa864bSJoonsoo Kim 348eefa864bSJoonsoo Kim static int __meminit page_ext_callback(struct notifier_block *self, 349eefa864bSJoonsoo Kim unsigned long action, void *arg) 350eefa864bSJoonsoo Kim { 351eefa864bSJoonsoo Kim struct memory_notify *mn = arg; 352eefa864bSJoonsoo Kim int ret = 0; 353eefa864bSJoonsoo Kim 354eefa864bSJoonsoo Kim switch (action) { 355eefa864bSJoonsoo Kim case MEM_GOING_ONLINE: 356eefa864bSJoonsoo Kim ret = online_page_ext(mn->start_pfn, 357eefa864bSJoonsoo Kim mn->nr_pages, mn->status_change_nid); 358eefa864bSJoonsoo Kim break; 359eefa864bSJoonsoo Kim case MEM_OFFLINE: 360eefa864bSJoonsoo Kim offline_page_ext(mn->start_pfn, 361eefa864bSJoonsoo Kim mn->nr_pages, mn->status_change_nid); 362eefa864bSJoonsoo Kim break; 363eefa864bSJoonsoo Kim case MEM_CANCEL_ONLINE: 364eefa864bSJoonsoo Kim offline_page_ext(mn->start_pfn, 365eefa864bSJoonsoo Kim mn->nr_pages, mn->status_change_nid); 366eefa864bSJoonsoo Kim break; 367eefa864bSJoonsoo Kim case MEM_GOING_OFFLINE: 368eefa864bSJoonsoo Kim break; 369eefa864bSJoonsoo Kim case MEM_ONLINE: 370eefa864bSJoonsoo Kim case MEM_CANCEL_OFFLINE: 371eefa864bSJoonsoo Kim break; 372eefa864bSJoonsoo Kim } 373eefa864bSJoonsoo Kim 374eefa864bSJoonsoo Kim return notifier_from_errno(ret); 375eefa864bSJoonsoo Kim } 376eefa864bSJoonsoo Kim 377eefa864bSJoonsoo Kim #endif 378eefa864bSJoonsoo Kim 379eefa864bSJoonsoo Kim void __init page_ext_init(void) 380eefa864bSJoonsoo Kim { 381eefa864bSJoonsoo Kim unsigned long pfn; 382eefa864bSJoonsoo Kim int nid; 383eefa864bSJoonsoo Kim 384eefa864bSJoonsoo Kim if (!invoke_need_callbacks()) 385eefa864bSJoonsoo Kim return; 386eefa864bSJoonsoo Kim 387eefa864bSJoonsoo Kim for_each_node_state(nid, N_MEMORY) { 388eefa864bSJoonsoo Kim unsigned long start_pfn, end_pfn; 389eefa864bSJoonsoo Kim 390eefa864bSJoonsoo Kim start_pfn = node_start_pfn(nid); 391eefa864bSJoonsoo Kim end_pfn = node_end_pfn(nid); 392eefa864bSJoonsoo Kim /* 393eefa864bSJoonsoo Kim * start_pfn and end_pfn may not be aligned to SECTION and the 394eefa864bSJoonsoo Kim * page->flags of out of node pages are not initialized. So we 395eefa864bSJoonsoo Kim * scan [start_pfn, the biggest section's pfn < end_pfn) here. 396eefa864bSJoonsoo Kim */ 397eefa864bSJoonsoo Kim for (pfn = start_pfn; pfn < end_pfn; 398eefa864bSJoonsoo Kim pfn = ALIGN(pfn + 1, PAGES_PER_SECTION)) { 399eefa864bSJoonsoo Kim 400eefa864bSJoonsoo Kim if (!pfn_valid(pfn)) 401eefa864bSJoonsoo Kim continue; 402eefa864bSJoonsoo Kim /* 403eefa864bSJoonsoo Kim * Nodes's pfns can be overlapping. 404eefa864bSJoonsoo Kim * We know some arch can have a nodes layout such as 405eefa864bSJoonsoo Kim * -------------pfn--------------> 406eefa864bSJoonsoo Kim * N0 | N1 | N2 | N0 | N1 | N2|.... 407eefa864bSJoonsoo Kim */ 4082f1ee091SQian Cai if (pfn_to_nid(pfn) != nid) 409eefa864bSJoonsoo Kim continue; 410eefa864bSJoonsoo Kim if (init_section_page_ext(pfn, nid)) 411eefa864bSJoonsoo Kim goto oom; 4120fc542b7SVlastimil Babka cond_resched(); 413eefa864bSJoonsoo Kim } 414eefa864bSJoonsoo Kim } 415eefa864bSJoonsoo Kim hotplug_memory_notifier(page_ext_callback, 0); 416eefa864bSJoonsoo Kim pr_info("allocated %ld bytes of page_ext\n", total_usage); 417eefa864bSJoonsoo Kim invoke_init_callbacks(); 418eefa864bSJoonsoo Kim return; 419eefa864bSJoonsoo Kim 420eefa864bSJoonsoo Kim oom: 421eefa864bSJoonsoo Kim panic("Out of memory"); 422eefa864bSJoonsoo Kim } 423eefa864bSJoonsoo Kim 424eefa864bSJoonsoo Kim void __meminit pgdat_page_ext_init(struct pglist_data *pgdat) 425eefa864bSJoonsoo Kim { 426eefa864bSJoonsoo Kim } 427eefa864bSJoonsoo Kim 428eefa864bSJoonsoo Kim #endif 429