1457c8996SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 23947be19SDave Hansen /* 33947be19SDave Hansen * linux/mm/memory_hotplug.c 43947be19SDave Hansen * 53947be19SDave Hansen * Copyright (C) 63947be19SDave Hansen */ 73947be19SDave Hansen 83947be19SDave Hansen #include <linux/stddef.h> 93947be19SDave Hansen #include <linux/mm.h> 10174cd4b1SIngo Molnar #include <linux/sched/signal.h> 113947be19SDave Hansen #include <linux/swap.h> 123947be19SDave Hansen #include <linux/interrupt.h> 133947be19SDave Hansen #include <linux/pagemap.h> 143947be19SDave Hansen #include <linux/compiler.h> 15b95f1b31SPaul Gortmaker #include <linux/export.h> 163947be19SDave Hansen #include <linux/pagevec.h> 172d1d43f6SChandra Seetharaman #include <linux/writeback.h> 183947be19SDave Hansen #include <linux/slab.h> 193947be19SDave Hansen #include <linux/sysctl.h> 203947be19SDave Hansen #include <linux/cpu.h> 213947be19SDave Hansen #include <linux/memory.h> 224b94ffdcSDan Williams #include <linux/memremap.h> 233947be19SDave Hansen #include <linux/memory_hotplug.h> 243947be19SDave Hansen #include <linux/highmem.h> 253947be19SDave Hansen #include <linux/vmalloc.h> 260a547039SKAMEZAWA Hiroyuki #include <linux/ioport.h> 270c0e6195SKAMEZAWA Hiroyuki #include <linux/delay.h> 280c0e6195SKAMEZAWA Hiroyuki #include <linux/migrate.h> 290c0e6195SKAMEZAWA Hiroyuki #include <linux/page-isolation.h> 3071088785SBadari Pulavarty #include <linux/pfn.h> 316ad696d2SAndi Kleen #include <linux/suspend.h> 326d9c285aSKOSAKI Motohiro #include <linux/mm_inline.h> 33d96ae530Sakpm@linux-foundation.org #include <linux/firmware-map.h> 3460a5a19eSTang Chen #include <linux/stop_machine.h> 35c8721bbbSNaoya Horiguchi #include <linux/hugetlb.h> 36c5320926STang Chen #include <linux/memblock.h> 37698b1b30SVlastimil Babka #include <linux/compaction.h> 38b15c8726SMichal Hocko #include <linux/rmap.h> 393947be19SDave Hansen 403947be19SDave Hansen #include <asm/tlbflush.h> 413947be19SDave Hansen 421e5ad9a3SAdrian Bunk #include "internal.h" 43e900a918SDan Williams #include "shuffle.h" 441e5ad9a3SAdrian Bunk 459d0ad8caSDaniel Kiper /* 469d0ad8caSDaniel Kiper * online_page_callback contains pointer to current page onlining function. 479d0ad8caSDaniel Kiper * Initially it is generic_online_page(). If it is required it could be 489d0ad8caSDaniel Kiper * changed by calling set_online_page_callback() for callback registration 499d0ad8caSDaniel Kiper * and restore_online_page_callback() for generic callback restore. 509d0ad8caSDaniel Kiper */ 519d0ad8caSDaniel Kiper 52a9cd410aSArun KS static void generic_online_page(struct page *page, unsigned int order); 539d0ad8caSDaniel Kiper 549d0ad8caSDaniel Kiper static online_page_callback_t online_page_callback = generic_online_page; 55bfc8c901SVladimir Davydov static DEFINE_MUTEX(online_page_callback_lock); 569d0ad8caSDaniel Kiper 573f906ba2SThomas Gleixner DEFINE_STATIC_PERCPU_RWSEM(mem_hotplug_lock); 5820d6c96bSKOSAKI Motohiro 593f906ba2SThomas Gleixner void get_online_mems(void) 603f906ba2SThomas Gleixner { 613f906ba2SThomas Gleixner percpu_down_read(&mem_hotplug_lock); 623f906ba2SThomas Gleixner } 63bfc8c901SVladimir Davydov 643f906ba2SThomas Gleixner void put_online_mems(void) 653f906ba2SThomas Gleixner { 663f906ba2SThomas Gleixner percpu_up_read(&mem_hotplug_lock); 673f906ba2SThomas Gleixner } 68bfc8c901SVladimir Davydov 694932381eSMichal Hocko bool movable_node_enabled = false; 704932381eSMichal Hocko 718604d9e5SVitaly Kuznetsov #ifndef CONFIG_MEMORY_HOTPLUG_DEFAULT_ONLINE 7231bc3858SVitaly Kuznetsov bool memhp_auto_online; 738604d9e5SVitaly Kuznetsov #else 748604d9e5SVitaly Kuznetsov bool memhp_auto_online = true; 758604d9e5SVitaly Kuznetsov #endif 7631bc3858SVitaly Kuznetsov EXPORT_SYMBOL_GPL(memhp_auto_online); 7731bc3858SVitaly Kuznetsov 7886dd995dSVitaly Kuznetsov static int __init setup_memhp_default_state(char *str) 7986dd995dSVitaly Kuznetsov { 8086dd995dSVitaly Kuznetsov if (!strcmp(str, "online")) 8186dd995dSVitaly Kuznetsov memhp_auto_online = true; 8286dd995dSVitaly Kuznetsov else if (!strcmp(str, "offline")) 8386dd995dSVitaly Kuznetsov memhp_auto_online = false; 8486dd995dSVitaly Kuznetsov 8586dd995dSVitaly Kuznetsov return 1; 8686dd995dSVitaly Kuznetsov } 8786dd995dSVitaly Kuznetsov __setup("memhp_default_state=", setup_memhp_default_state); 8886dd995dSVitaly Kuznetsov 8930467e0bSDavid Rientjes void mem_hotplug_begin(void) 90bfc8c901SVladimir Davydov { 913f906ba2SThomas Gleixner cpus_read_lock(); 923f906ba2SThomas Gleixner percpu_down_write(&mem_hotplug_lock); 93bfc8c901SVladimir Davydov } 94bfc8c901SVladimir Davydov 9530467e0bSDavid Rientjes void mem_hotplug_done(void) 96bfc8c901SVladimir Davydov { 973f906ba2SThomas Gleixner percpu_up_write(&mem_hotplug_lock); 983f906ba2SThomas Gleixner cpus_read_unlock(); 99bfc8c901SVladimir Davydov } 10020d6c96bSKOSAKI Motohiro 101357b4da5SJuergen Gross u64 max_mem_size = U64_MAX; 102357b4da5SJuergen Gross 10345e0b78bSKeith Mannthey /* add this memory to iomem resource */ 10445e0b78bSKeith Mannthey static struct resource *register_memory_resource(u64 start, u64 size) 10545e0b78bSKeith Mannthey { 1062794129eSDave Hansen struct resource *res; 1072794129eSDave Hansen unsigned long flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY; 1082794129eSDave Hansen char *resource_name = "System RAM"; 109357b4da5SJuergen Gross 110357b4da5SJuergen Gross if (start + size > max_mem_size) 111357b4da5SJuergen Gross return ERR_PTR(-E2BIG); 112357b4da5SJuergen Gross 1132794129eSDave Hansen /* 1142794129eSDave Hansen * Request ownership of the new memory range. This might be 1152794129eSDave Hansen * a child of an existing resource that was present but 1162794129eSDave Hansen * not marked as busy. 1172794129eSDave Hansen */ 1182794129eSDave Hansen res = __request_region(&iomem_resource, start, size, 1192794129eSDave Hansen resource_name, flags); 12045e0b78bSKeith Mannthey 1212794129eSDave Hansen if (!res) { 1222794129eSDave Hansen pr_debug("Unable to reserve System RAM region: %016llx->%016llx\n", 1232794129eSDave Hansen start, start + size); 1246f754ba4SVitaly Kuznetsov return ERR_PTR(-EEXIST); 12545e0b78bSKeith Mannthey } 12645e0b78bSKeith Mannthey return res; 12745e0b78bSKeith Mannthey } 12845e0b78bSKeith Mannthey 12945e0b78bSKeith Mannthey static void release_memory_resource(struct resource *res) 13045e0b78bSKeith Mannthey { 13145e0b78bSKeith Mannthey if (!res) 13245e0b78bSKeith Mannthey return; 13345e0b78bSKeith Mannthey release_resource(res); 13445e0b78bSKeith Mannthey kfree(res); 13545e0b78bSKeith Mannthey } 13645e0b78bSKeith Mannthey 13753947027SKeith Mannthey #ifdef CONFIG_MEMORY_HOTPLUG_SPARSE 13846723bfaSYasuaki Ishimatsu void get_page_bootmem(unsigned long info, struct page *page, 1395f24ce5fSAndrea Arcangeli unsigned long type) 14004753278SYasunori Goto { 141ddffe98dSYasuaki Ishimatsu page->freelist = (void *)type; 14204753278SYasunori Goto SetPagePrivate(page); 14304753278SYasunori Goto set_page_private(page, info); 144fe896d18SJoonsoo Kim page_ref_inc(page); 14504753278SYasunori Goto } 14604753278SYasunori Goto 147170a5a7eSJiang Liu void put_page_bootmem(struct page *page) 14804753278SYasunori Goto { 1495f24ce5fSAndrea Arcangeli unsigned long type; 15004753278SYasunori Goto 151ddffe98dSYasuaki Ishimatsu type = (unsigned long) page->freelist; 1525f24ce5fSAndrea Arcangeli BUG_ON(type < MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE || 1535f24ce5fSAndrea Arcangeli type > MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE); 15404753278SYasunori Goto 155fe896d18SJoonsoo Kim if (page_ref_dec_return(page) == 1) { 156ddffe98dSYasuaki Ishimatsu page->freelist = NULL; 15704753278SYasunori Goto ClearPagePrivate(page); 15804753278SYasunori Goto set_page_private(page, 0); 1595f24ce5fSAndrea Arcangeli INIT_LIST_HEAD(&page->lru); 160170a5a7eSJiang Liu free_reserved_page(page); 16104753278SYasunori Goto } 16204753278SYasunori Goto } 16304753278SYasunori Goto 16446723bfaSYasuaki Ishimatsu #ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE 16546723bfaSYasuaki Ishimatsu #ifndef CONFIG_SPARSEMEM_VMEMMAP 166d92bc318SAdrian Bunk static void register_page_bootmem_info_section(unsigned long start_pfn) 16704753278SYasunori Goto { 168f1eca35aSDan Williams unsigned long mapsize, section_nr, i; 16904753278SYasunori Goto struct mem_section *ms; 17004753278SYasunori Goto struct page *page, *memmap; 171f1eca35aSDan Williams struct mem_section_usage *usage; 17204753278SYasunori Goto 17304753278SYasunori Goto section_nr = pfn_to_section_nr(start_pfn); 17404753278SYasunori Goto ms = __nr_to_section(section_nr); 17504753278SYasunori Goto 17604753278SYasunori Goto /* Get section's memmap address */ 17704753278SYasunori Goto memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr); 17804753278SYasunori Goto 17904753278SYasunori Goto /* 18004753278SYasunori Goto * Get page for the memmap's phys address 18104753278SYasunori Goto * XXX: need more consideration for sparse_vmemmap... 18204753278SYasunori Goto */ 18304753278SYasunori Goto page = virt_to_page(memmap); 18404753278SYasunori Goto mapsize = sizeof(struct page) * PAGES_PER_SECTION; 18504753278SYasunori Goto mapsize = PAGE_ALIGN(mapsize) >> PAGE_SHIFT; 18604753278SYasunori Goto 18704753278SYasunori Goto /* remember memmap's page */ 18804753278SYasunori Goto for (i = 0; i < mapsize; i++, page++) 18904753278SYasunori Goto get_page_bootmem(section_nr, page, SECTION_INFO); 19004753278SYasunori Goto 191f1eca35aSDan Williams usage = ms->usage; 192f1eca35aSDan Williams page = virt_to_page(usage); 19304753278SYasunori Goto 194f1eca35aSDan Williams mapsize = PAGE_ALIGN(mem_section_usage_size()) >> PAGE_SHIFT; 19504753278SYasunori Goto 19604753278SYasunori Goto for (i = 0; i < mapsize; i++, page++) 197af370fb8SYasunori Goto get_page_bootmem(section_nr, page, MIX_SECTION_INFO); 19804753278SYasunori Goto 19904753278SYasunori Goto } 20046723bfaSYasuaki Ishimatsu #else /* CONFIG_SPARSEMEM_VMEMMAP */ 20146723bfaSYasuaki Ishimatsu static void register_page_bootmem_info_section(unsigned long start_pfn) 20246723bfaSYasuaki Ishimatsu { 203f1eca35aSDan Williams unsigned long mapsize, section_nr, i; 20446723bfaSYasuaki Ishimatsu struct mem_section *ms; 20546723bfaSYasuaki Ishimatsu struct page *page, *memmap; 206f1eca35aSDan Williams struct mem_section_usage *usage; 20746723bfaSYasuaki Ishimatsu 20846723bfaSYasuaki Ishimatsu section_nr = pfn_to_section_nr(start_pfn); 20946723bfaSYasuaki Ishimatsu ms = __nr_to_section(section_nr); 21046723bfaSYasuaki Ishimatsu 21146723bfaSYasuaki Ishimatsu memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr); 21246723bfaSYasuaki Ishimatsu 21346723bfaSYasuaki Ishimatsu register_page_bootmem_memmap(section_nr, memmap, PAGES_PER_SECTION); 21446723bfaSYasuaki Ishimatsu 215f1eca35aSDan Williams usage = ms->usage; 216f1eca35aSDan Williams page = virt_to_page(usage); 21746723bfaSYasuaki Ishimatsu 218f1eca35aSDan Williams mapsize = PAGE_ALIGN(mem_section_usage_size()) >> PAGE_SHIFT; 21946723bfaSYasuaki Ishimatsu 22046723bfaSYasuaki Ishimatsu for (i = 0; i < mapsize; i++, page++) 22146723bfaSYasuaki Ishimatsu get_page_bootmem(section_nr, page, MIX_SECTION_INFO); 22246723bfaSYasuaki Ishimatsu } 22346723bfaSYasuaki Ishimatsu #endif /* !CONFIG_SPARSEMEM_VMEMMAP */ 22404753278SYasunori Goto 2257ded384aSLinus Torvalds void __init register_page_bootmem_info_node(struct pglist_data *pgdat) 22604753278SYasunori Goto { 22704753278SYasunori Goto unsigned long i, pfn, end_pfn, nr_pages; 22804753278SYasunori Goto int node = pgdat->node_id; 22904753278SYasunori Goto struct page *page; 23004753278SYasunori Goto 23104753278SYasunori Goto nr_pages = PAGE_ALIGN(sizeof(struct pglist_data)) >> PAGE_SHIFT; 23204753278SYasunori Goto page = virt_to_page(pgdat); 23304753278SYasunori Goto 23404753278SYasunori Goto for (i = 0; i < nr_pages; i++, page++) 23504753278SYasunori Goto get_page_bootmem(node, page, NODE_INFO); 23604753278SYasunori Goto 23704753278SYasunori Goto pfn = pgdat->node_start_pfn; 238c1f19495SCody P Schafer end_pfn = pgdat_end_pfn(pgdat); 23904753278SYasunori Goto 2407e9f5eb0STang Chen /* register section info */ 241f14851afSqiuxishi for (; pfn < end_pfn; pfn += PAGES_PER_SECTION) { 242f14851afSqiuxishi /* 243f14851afSqiuxishi * Some platforms can assign the same pfn to multiple nodes - on 244f14851afSqiuxishi * node0 as well as nodeN. To avoid registering a pfn against 245f14851afSqiuxishi * multiple nodes we check that this pfn does not already 2467e9f5eb0STang Chen * reside in some other nodes. 247f14851afSqiuxishi */ 248f65e91dfSYang Shi if (pfn_valid(pfn) && (early_pfn_to_nid(pfn) == node)) 24904753278SYasunori Goto register_page_bootmem_info_section(pfn); 250f14851afSqiuxishi } 25104753278SYasunori Goto } 25246723bfaSYasuaki Ishimatsu #endif /* CONFIG_HAVE_BOOTMEM_INFO_NODE */ 25304753278SYasunori Goto 2547ea62160SDan Williams static int check_pfn_span(unsigned long pfn, unsigned long nr_pages, 2557ea62160SDan Williams const char *reason) 2567ea62160SDan Williams { 2577ea62160SDan Williams /* 2587ea62160SDan Williams * Disallow all operations smaller than a sub-section and only 2597ea62160SDan Williams * allow operations smaller than a section for 2607ea62160SDan Williams * SPARSEMEM_VMEMMAP. Note that check_hotplug_memory_range() 2617ea62160SDan Williams * enforces a larger memory_block_size_bytes() granularity for 2627ea62160SDan Williams * memory that will be marked online, so this check should only 2637ea62160SDan Williams * fire for direct arch_{add,remove}_memory() users outside of 2647ea62160SDan Williams * add_memory_resource(). 2657ea62160SDan Williams */ 2667ea62160SDan Williams unsigned long min_align; 2677ea62160SDan Williams 2687ea62160SDan Williams if (IS_ENABLED(CONFIG_SPARSEMEM_VMEMMAP)) 2697ea62160SDan Williams min_align = PAGES_PER_SUBSECTION; 2707ea62160SDan Williams else 2717ea62160SDan Williams min_align = PAGES_PER_SECTION; 2727ea62160SDan Williams if (!IS_ALIGNED(pfn, min_align) 2737ea62160SDan Williams || !IS_ALIGNED(nr_pages, min_align)) { 2747ea62160SDan Williams WARN(1, "Misaligned __%s_pages start: %#lx end: #%lx\n", 2757ea62160SDan Williams reason, pfn, pfn + nr_pages - 1); 2767ea62160SDan Williams return -EINVAL; 2777ea62160SDan Williams } 2787ea62160SDan Williams return 0; 2797ea62160SDan Williams } 2807ea62160SDan Williams 2814edd7cefSDavid Rientjes /* 2824edd7cefSDavid Rientjes * Reasonably generic function for adding memory. It is 2834edd7cefSDavid Rientjes * expected that archs that support memory hotplug will 2844edd7cefSDavid Rientjes * call this function after deciding the zone to which to 2854edd7cefSDavid Rientjes * add the new pages. 2864edd7cefSDavid Rientjes */ 2877ea62160SDan Williams int __ref __add_pages(int nid, unsigned long pfn, unsigned long nr_pages, 2887ea62160SDan Williams struct mhp_restrictions *restrictions) 2894edd7cefSDavid Rientjes { 2909a845030SDan Williams int err; 2919a845030SDan Williams unsigned long nr, start_sec, end_sec; 292940519f0SMichal Hocko struct vmem_altmap *altmap = restrictions->altmap; 2934b94ffdcSDan Williams 2944b94ffdcSDan Williams if (altmap) { 2954b94ffdcSDan Williams /* 2964b94ffdcSDan Williams * Validate altmap is within bounds of the total request 2974b94ffdcSDan Williams */ 2987ea62160SDan Williams if (altmap->base_pfn != pfn 2994b94ffdcSDan Williams || vmem_altmap_offset(altmap) > nr_pages) { 3004b94ffdcSDan Williams pr_warn_once("memory add fail, invalid altmap\n"); 3017ea62160SDan Williams return -EINVAL; 3024b94ffdcSDan Williams } 3034b94ffdcSDan Williams altmap->alloc = 0; 3044b94ffdcSDan Williams } 3054b94ffdcSDan Williams 3067ea62160SDan Williams err = check_pfn_span(pfn, nr_pages, "add"); 3077ea62160SDan Williams if (err) 3087ea62160SDan Williams return err; 3097ea62160SDan Williams 3107ea62160SDan Williams start_sec = pfn_to_section_nr(pfn); 3117ea62160SDan Williams end_sec = pfn_to_section_nr(pfn + nr_pages - 1); 3129a845030SDan Williams for (nr = start_sec; nr <= end_sec; nr++) { 3137ea62160SDan Williams unsigned long pfns; 3147ea62160SDan Williams 3157ea62160SDan Williams pfns = min(nr_pages, PAGES_PER_SECTION 3167ea62160SDan Williams - (pfn & ~PAGE_SECTION_MASK)); 317ba72b4c8SDan Williams err = sparse_add_section(nid, pfn, pfns, altmap); 318ba72b4c8SDan Williams if (err) 319ba72b4c8SDan Williams break; 3207ea62160SDan Williams pfn += pfns; 3217ea62160SDan Williams nr_pages -= pfns; 322f64ac5e6SMichal Hocko cond_resched(); 3234edd7cefSDavid Rientjes } 324c435a390SZhu Guihua vmemmap_populate_print_last(); 3254edd7cefSDavid Rientjes return err; 3264edd7cefSDavid Rientjes } 3274edd7cefSDavid Rientjes 328815121d2SYasuaki Ishimatsu /* find the smallest valid pfn in the range [start_pfn, end_pfn) */ 329d09b0137SYASUAKI ISHIMATSU static unsigned long find_smallest_section_pfn(int nid, struct zone *zone, 330815121d2SYasuaki Ishimatsu unsigned long start_pfn, 331815121d2SYasuaki Ishimatsu unsigned long end_pfn) 332815121d2SYasuaki Ishimatsu { 33349ba3c6bSDan Williams for (; start_pfn < end_pfn; start_pfn += PAGES_PER_SUBSECTION) { 3347ce700bfSDavid Hildenbrand if (unlikely(!pfn_to_online_page(start_pfn))) 335815121d2SYasuaki Ishimatsu continue; 336815121d2SYasuaki Ishimatsu 337815121d2SYasuaki Ishimatsu if (unlikely(pfn_to_nid(start_pfn) != nid)) 338815121d2SYasuaki Ishimatsu continue; 339815121d2SYasuaki Ishimatsu 340815121d2SYasuaki Ishimatsu if (zone && zone != page_zone(pfn_to_page(start_pfn))) 341815121d2SYasuaki Ishimatsu continue; 342815121d2SYasuaki Ishimatsu 343815121d2SYasuaki Ishimatsu return start_pfn; 344815121d2SYasuaki Ishimatsu } 345815121d2SYasuaki Ishimatsu 346815121d2SYasuaki Ishimatsu return 0; 347815121d2SYasuaki Ishimatsu } 348815121d2SYasuaki Ishimatsu 349815121d2SYasuaki Ishimatsu /* find the biggest valid pfn in the range [start_pfn, end_pfn). */ 350d09b0137SYASUAKI ISHIMATSU static unsigned long find_biggest_section_pfn(int nid, struct zone *zone, 351815121d2SYasuaki Ishimatsu unsigned long start_pfn, 352815121d2SYasuaki Ishimatsu unsigned long end_pfn) 353815121d2SYasuaki Ishimatsu { 354815121d2SYasuaki Ishimatsu unsigned long pfn; 355815121d2SYasuaki Ishimatsu 356815121d2SYasuaki Ishimatsu /* pfn is the end pfn of a memory section. */ 357815121d2SYasuaki Ishimatsu pfn = end_pfn - 1; 35849ba3c6bSDan Williams for (; pfn >= start_pfn; pfn -= PAGES_PER_SUBSECTION) { 3597ce700bfSDavid Hildenbrand if (unlikely(!pfn_to_online_page(pfn))) 360815121d2SYasuaki Ishimatsu continue; 361815121d2SYasuaki Ishimatsu 362815121d2SYasuaki Ishimatsu if (unlikely(pfn_to_nid(pfn) != nid)) 363815121d2SYasuaki Ishimatsu continue; 364815121d2SYasuaki Ishimatsu 365815121d2SYasuaki Ishimatsu if (zone && zone != page_zone(pfn_to_page(pfn))) 366815121d2SYasuaki Ishimatsu continue; 367815121d2SYasuaki Ishimatsu 368815121d2SYasuaki Ishimatsu return pfn; 369815121d2SYasuaki Ishimatsu } 370815121d2SYasuaki Ishimatsu 371815121d2SYasuaki Ishimatsu return 0; 372815121d2SYasuaki Ishimatsu } 373815121d2SYasuaki Ishimatsu 374815121d2SYasuaki Ishimatsu static void shrink_zone_span(struct zone *zone, unsigned long start_pfn, 375815121d2SYasuaki Ishimatsu unsigned long end_pfn) 376815121d2SYasuaki Ishimatsu { 377815121d2SYasuaki Ishimatsu unsigned long zone_start_pfn = zone->zone_start_pfn; 378c33bc315SXishi Qiu unsigned long z = zone_end_pfn(zone); /* zone_end_pfn namespace clash */ 379c33bc315SXishi Qiu unsigned long zone_end_pfn = z; 380815121d2SYasuaki Ishimatsu unsigned long pfn; 381815121d2SYasuaki Ishimatsu int nid = zone_to_nid(zone); 382815121d2SYasuaki Ishimatsu 383815121d2SYasuaki Ishimatsu zone_span_writelock(zone); 384815121d2SYasuaki Ishimatsu if (zone_start_pfn == start_pfn) { 385815121d2SYasuaki Ishimatsu /* 386815121d2SYasuaki Ishimatsu * If the section is smallest section in the zone, it need 387815121d2SYasuaki Ishimatsu * shrink zone->zone_start_pfn and zone->zone_spanned_pages. 388815121d2SYasuaki Ishimatsu * In this case, we find second smallest valid mem_section 389815121d2SYasuaki Ishimatsu * for shrinking zone. 390815121d2SYasuaki Ishimatsu */ 391815121d2SYasuaki Ishimatsu pfn = find_smallest_section_pfn(nid, zone, end_pfn, 392815121d2SYasuaki Ishimatsu zone_end_pfn); 393815121d2SYasuaki Ishimatsu if (pfn) { 394815121d2SYasuaki Ishimatsu zone->zone_start_pfn = pfn; 395815121d2SYasuaki Ishimatsu zone->spanned_pages = zone_end_pfn - pfn; 396815121d2SYasuaki Ishimatsu } 397815121d2SYasuaki Ishimatsu } else if (zone_end_pfn == end_pfn) { 398815121d2SYasuaki Ishimatsu /* 399815121d2SYasuaki Ishimatsu * If the section is biggest section in the zone, it need 400815121d2SYasuaki Ishimatsu * shrink zone->spanned_pages. 401815121d2SYasuaki Ishimatsu * In this case, we find second biggest valid mem_section for 402815121d2SYasuaki Ishimatsu * shrinking zone. 403815121d2SYasuaki Ishimatsu */ 404815121d2SYasuaki Ishimatsu pfn = find_biggest_section_pfn(nid, zone, zone_start_pfn, 405815121d2SYasuaki Ishimatsu start_pfn); 406815121d2SYasuaki Ishimatsu if (pfn) 407815121d2SYasuaki Ishimatsu zone->spanned_pages = pfn - zone_start_pfn + 1; 408815121d2SYasuaki Ishimatsu } 409815121d2SYasuaki Ishimatsu 410815121d2SYasuaki Ishimatsu /* 411815121d2SYasuaki Ishimatsu * The section is not biggest or smallest mem_section in the zone, it 412815121d2SYasuaki Ishimatsu * only creates a hole in the zone. So in this case, we need not 413815121d2SYasuaki Ishimatsu * change the zone. But perhaps, the zone has only hole data. Thus 414815121d2SYasuaki Ishimatsu * it check the zone has only hole or not. 415815121d2SYasuaki Ishimatsu */ 416815121d2SYasuaki Ishimatsu pfn = zone_start_pfn; 41749ba3c6bSDan Williams for (; pfn < zone_end_pfn; pfn += PAGES_PER_SUBSECTION) { 4187ce700bfSDavid Hildenbrand if (unlikely(!pfn_to_online_page(pfn))) 419815121d2SYasuaki Ishimatsu continue; 420815121d2SYasuaki Ishimatsu 421815121d2SYasuaki Ishimatsu if (page_zone(pfn_to_page(pfn)) != zone) 422815121d2SYasuaki Ishimatsu continue; 423815121d2SYasuaki Ishimatsu 42449ba3c6bSDan Williams /* Skip range to be removed */ 42549ba3c6bSDan Williams if (pfn >= start_pfn && pfn < end_pfn) 426815121d2SYasuaki Ishimatsu continue; 427815121d2SYasuaki Ishimatsu 428815121d2SYasuaki Ishimatsu /* If we find valid section, we have nothing to do */ 429815121d2SYasuaki Ishimatsu zone_span_writeunlock(zone); 430815121d2SYasuaki Ishimatsu return; 431815121d2SYasuaki Ishimatsu } 432815121d2SYasuaki Ishimatsu 433815121d2SYasuaki Ishimatsu /* The zone has no valid section */ 434815121d2SYasuaki Ishimatsu zone->zone_start_pfn = 0; 435815121d2SYasuaki Ishimatsu zone->spanned_pages = 0; 436815121d2SYasuaki Ishimatsu zone_span_writeunlock(zone); 437815121d2SYasuaki Ishimatsu } 438815121d2SYasuaki Ishimatsu 43900d6c019SDavid Hildenbrand static void update_pgdat_span(struct pglist_data *pgdat) 440815121d2SYasuaki Ishimatsu { 44100d6c019SDavid Hildenbrand unsigned long node_start_pfn = 0, node_end_pfn = 0; 44200d6c019SDavid Hildenbrand struct zone *zone; 443815121d2SYasuaki Ishimatsu 44400d6c019SDavid Hildenbrand for (zone = pgdat->node_zones; 44500d6c019SDavid Hildenbrand zone < pgdat->node_zones + MAX_NR_ZONES; zone++) { 44600d6c019SDavid Hildenbrand unsigned long zone_end_pfn = zone->zone_start_pfn + 44700d6c019SDavid Hildenbrand zone->spanned_pages; 44800d6c019SDavid Hildenbrand 44900d6c019SDavid Hildenbrand /* No need to lock the zones, they can't change. */ 450656d5711SDavid Hildenbrand if (!zone->spanned_pages) 451656d5711SDavid Hildenbrand continue; 452656d5711SDavid Hildenbrand if (!node_end_pfn) { 453656d5711SDavid Hildenbrand node_start_pfn = zone->zone_start_pfn; 454656d5711SDavid Hildenbrand node_end_pfn = zone_end_pfn; 455656d5711SDavid Hildenbrand continue; 456656d5711SDavid Hildenbrand } 457656d5711SDavid Hildenbrand 45800d6c019SDavid Hildenbrand if (zone_end_pfn > node_end_pfn) 45900d6c019SDavid Hildenbrand node_end_pfn = zone_end_pfn; 46000d6c019SDavid Hildenbrand if (zone->zone_start_pfn < node_start_pfn) 46100d6c019SDavid Hildenbrand node_start_pfn = zone->zone_start_pfn; 462815121d2SYasuaki Ishimatsu } 463815121d2SYasuaki Ishimatsu 46400d6c019SDavid Hildenbrand pgdat->node_start_pfn = node_start_pfn; 46500d6c019SDavid Hildenbrand pgdat->node_spanned_pages = node_end_pfn - node_start_pfn; 466815121d2SYasuaki Ishimatsu } 467815121d2SYasuaki Ishimatsu 4687ea62160SDan Williams static void __remove_zone(struct zone *zone, unsigned long start_pfn, 4697ea62160SDan Williams unsigned long nr_pages) 470815121d2SYasuaki Ishimatsu { 471815121d2SYasuaki Ishimatsu struct pglist_data *pgdat = zone->zone_pgdat; 472815121d2SYasuaki Ishimatsu unsigned long flags; 473815121d2SYasuaki Ishimatsu 4747ce700bfSDavid Hildenbrand #ifdef CONFIG_ZONE_DEVICE 4757ce700bfSDavid Hildenbrand /* 4767ce700bfSDavid Hildenbrand * Zone shrinking code cannot properly deal with ZONE_DEVICE. So 4777ce700bfSDavid Hildenbrand * we will not try to shrink the zones - which is okay as 4787ce700bfSDavid Hildenbrand * set_zone_contiguous() cannot deal with ZONE_DEVICE either way. 4797ce700bfSDavid Hildenbrand */ 4807ce700bfSDavid Hildenbrand if (zone_idx(zone) == ZONE_DEVICE) 4817ce700bfSDavid Hildenbrand return; 4827ce700bfSDavid Hildenbrand #endif 4837ce700bfSDavid Hildenbrand 484815121d2SYasuaki Ishimatsu pgdat_resize_lock(zone->zone_pgdat, &flags); 485815121d2SYasuaki Ishimatsu shrink_zone_span(zone, start_pfn, start_pfn + nr_pages); 48600d6c019SDavid Hildenbrand update_pgdat_span(pgdat); 487815121d2SYasuaki Ishimatsu pgdat_resize_unlock(zone->zone_pgdat, &flags); 488815121d2SYasuaki Ishimatsu } 489815121d2SYasuaki Ishimatsu 4907ea62160SDan Williams static void __remove_section(struct zone *zone, unsigned long pfn, 4917ea62160SDan Williams unsigned long nr_pages, unsigned long map_offset, 4929d1d887dSDavid Hildenbrand struct vmem_altmap *altmap) 493ea01ea93SBadari Pulavarty { 4947ea62160SDan Williams struct mem_section *ms = __nr_to_section(pfn_to_section_nr(pfn)); 495ea01ea93SBadari Pulavarty 4969d1d887dSDavid Hildenbrand if (WARN_ON_ONCE(!valid_section(ms))) 4979d1d887dSDavid Hildenbrand return; 498ea01ea93SBadari Pulavarty 4997ea62160SDan Williams __remove_zone(zone, pfn, nr_pages); 500ba72b4c8SDan Williams sparse_remove_section(ms, pfn, nr_pages, map_offset, altmap); 501ea01ea93SBadari Pulavarty } 502ea01ea93SBadari Pulavarty 503ea01ea93SBadari Pulavarty /** 504ea01ea93SBadari Pulavarty * __remove_pages() - remove sections of pages from a zone 505ea01ea93SBadari Pulavarty * @zone: zone from which pages need to be removed 5067ea62160SDan Williams * @pfn: starting pageframe (must be aligned to start of a section) 507ea01ea93SBadari Pulavarty * @nr_pages: number of pages to remove (must be multiple of section size) 508e8b098fcSMike Rapoport * @altmap: alternative device page map or %NULL if default memmap is used 509ea01ea93SBadari Pulavarty * 510ea01ea93SBadari Pulavarty * Generic helper function to remove section mappings and sysfs entries 511ea01ea93SBadari Pulavarty * for the section of the memory we are removing. Caller needs to make 512ea01ea93SBadari Pulavarty * sure that pages are marked reserved and zones are adjust properly by 513ea01ea93SBadari Pulavarty * calling offline_pages(). 514ea01ea93SBadari Pulavarty */ 5157ea62160SDan Williams void __remove_pages(struct zone *zone, unsigned long pfn, 516da024512SChristoph Hellwig unsigned long nr_pages, struct vmem_altmap *altmap) 517ea01ea93SBadari Pulavarty { 5184b94ffdcSDan Williams unsigned long map_offset = 0; 5199a845030SDan Williams unsigned long nr, start_sec, end_sec; 5204b94ffdcSDan Williams 5214b94ffdcSDan Williams map_offset = vmem_altmap_offset(altmap); 522ea01ea93SBadari Pulavarty 5237cf91a98SJoonsoo Kim clear_zone_contiguous(zone); 5247cf91a98SJoonsoo Kim 5257ea62160SDan Williams if (check_pfn_span(pfn, nr_pages, "remove")) 5267ea62160SDan Williams return; 527ea01ea93SBadari Pulavarty 5287ea62160SDan Williams start_sec = pfn_to_section_nr(pfn); 5297ea62160SDan Williams end_sec = pfn_to_section_nr(pfn + nr_pages - 1); 5309a845030SDan Williams for (nr = start_sec; nr <= end_sec; nr++) { 5317ea62160SDan Williams unsigned long pfns; 5324b94ffdcSDan Williams 533dd33ad7bSMichal Hocko cond_resched(); 5347ea62160SDan Williams pfns = min(nr_pages, PAGES_PER_SECTION 5357ea62160SDan Williams - (pfn & ~PAGE_SECTION_MASK)); 5367ea62160SDan Williams __remove_section(zone, pfn, pfns, map_offset, altmap); 5377ea62160SDan Williams pfn += pfns; 5387ea62160SDan Williams nr_pages -= pfns; 5394b94ffdcSDan Williams map_offset = 0; 540ea01ea93SBadari Pulavarty } 5417cf91a98SJoonsoo Kim 5427cf91a98SJoonsoo Kim set_zone_contiguous(zone); 543ea01ea93SBadari Pulavarty } 544ea01ea93SBadari Pulavarty 5459d0ad8caSDaniel Kiper int set_online_page_callback(online_page_callback_t callback) 5469d0ad8caSDaniel Kiper { 5479d0ad8caSDaniel Kiper int rc = -EINVAL; 5489d0ad8caSDaniel Kiper 549bfc8c901SVladimir Davydov get_online_mems(); 550bfc8c901SVladimir Davydov mutex_lock(&online_page_callback_lock); 5519d0ad8caSDaniel Kiper 5529d0ad8caSDaniel Kiper if (online_page_callback == generic_online_page) { 5539d0ad8caSDaniel Kiper online_page_callback = callback; 5549d0ad8caSDaniel Kiper rc = 0; 5559d0ad8caSDaniel Kiper } 5569d0ad8caSDaniel Kiper 557bfc8c901SVladimir Davydov mutex_unlock(&online_page_callback_lock); 558bfc8c901SVladimir Davydov put_online_mems(); 5599d0ad8caSDaniel Kiper 5609d0ad8caSDaniel Kiper return rc; 5619d0ad8caSDaniel Kiper } 5629d0ad8caSDaniel Kiper EXPORT_SYMBOL_GPL(set_online_page_callback); 5639d0ad8caSDaniel Kiper 5649d0ad8caSDaniel Kiper int restore_online_page_callback(online_page_callback_t callback) 5659d0ad8caSDaniel Kiper { 5669d0ad8caSDaniel Kiper int rc = -EINVAL; 5679d0ad8caSDaniel Kiper 568bfc8c901SVladimir Davydov get_online_mems(); 569bfc8c901SVladimir Davydov mutex_lock(&online_page_callback_lock); 5709d0ad8caSDaniel Kiper 5719d0ad8caSDaniel Kiper if (online_page_callback == callback) { 5729d0ad8caSDaniel Kiper online_page_callback = generic_online_page; 5739d0ad8caSDaniel Kiper rc = 0; 5749d0ad8caSDaniel Kiper } 5759d0ad8caSDaniel Kiper 576bfc8c901SVladimir Davydov mutex_unlock(&online_page_callback_lock); 577bfc8c901SVladimir Davydov put_online_mems(); 5789d0ad8caSDaniel Kiper 5799d0ad8caSDaniel Kiper return rc; 5809d0ad8caSDaniel Kiper } 5819d0ad8caSDaniel Kiper EXPORT_SYMBOL_GPL(restore_online_page_callback); 5829d0ad8caSDaniel Kiper 5839d0ad8caSDaniel Kiper void __online_page_set_limits(struct page *page) 584180c06efSJeremy Fitzhardinge { 5859d0ad8caSDaniel Kiper } 5869d0ad8caSDaniel Kiper EXPORT_SYMBOL_GPL(__online_page_set_limits); 5879d0ad8caSDaniel Kiper 5889d0ad8caSDaniel Kiper void __online_page_increment_counters(struct page *page) 5899d0ad8caSDaniel Kiper { 5903dcc0571SJiang Liu adjust_managed_page_count(page, 1); 5919d0ad8caSDaniel Kiper } 5929d0ad8caSDaniel Kiper EXPORT_SYMBOL_GPL(__online_page_increment_counters); 593180c06efSJeremy Fitzhardinge 5949d0ad8caSDaniel Kiper void __online_page_free(struct page *page) 5959d0ad8caSDaniel Kiper { 5963dcc0571SJiang Liu __free_reserved_page(page); 597180c06efSJeremy Fitzhardinge } 5989d0ad8caSDaniel Kiper EXPORT_SYMBOL_GPL(__online_page_free); 5999d0ad8caSDaniel Kiper 600a9cd410aSArun KS static void generic_online_page(struct page *page, unsigned int order) 6019d0ad8caSDaniel Kiper { 602cd02cf1aSQian Cai kernel_map_pages(page, 1 << order, 1); 603a9cd410aSArun KS __free_pages_core(page, order); 604a9cd410aSArun KS totalram_pages_add(1UL << order); 605a9cd410aSArun KS #ifdef CONFIG_HIGHMEM 606a9cd410aSArun KS if (PageHighMem(page)) 607a9cd410aSArun KS totalhigh_pages_add(1UL << order); 608a9cd410aSArun KS #endif 609a9cd410aSArun KS } 610a9cd410aSArun KS 61175884fb1SKAMEZAWA Hiroyuki static int online_pages_range(unsigned long start_pfn, unsigned long nr_pages, 61275884fb1SKAMEZAWA Hiroyuki void *arg) 6133947be19SDave Hansen { 614b2c2ab20SDavid Hildenbrand const unsigned long end_pfn = start_pfn + nr_pages; 615b2c2ab20SDavid Hildenbrand unsigned long pfn; 616b2c2ab20SDavid Hildenbrand int order; 6172d070eabSMichal Hocko 618b2c2ab20SDavid Hildenbrand /* 619b2c2ab20SDavid Hildenbrand * Online the pages. The callback might decide to keep some pages 620b2c2ab20SDavid Hildenbrand * PG_reserved (to add them to the buddy later), but we still account 621b2c2ab20SDavid Hildenbrand * them as being online/belonging to this zone ("present"). 622b2c2ab20SDavid Hildenbrand */ 623b2c2ab20SDavid Hildenbrand for (pfn = start_pfn; pfn < end_pfn; pfn += 1ul << order) { 624b2c2ab20SDavid Hildenbrand order = min(MAX_ORDER - 1, get_order(PFN_PHYS(end_pfn - pfn))); 625bd02cc01SDavid Hildenbrand /* __free_pages_core() wants pfns to be aligned to the order */ 626bd02cc01SDavid Hildenbrand if (WARN_ON_ONCE(!IS_ALIGNED(pfn, 1ul << order))) 627bd02cc01SDavid Hildenbrand order = 0; 628b2c2ab20SDavid Hildenbrand (*online_page_callback)(pfn_to_page(pfn), order); 629b2c2ab20SDavid Hildenbrand } 6302d070eabSMichal Hocko 631b2c2ab20SDavid Hildenbrand /* mark all involved sections as online */ 632b2c2ab20SDavid Hildenbrand online_mem_sections(start_pfn, end_pfn); 633b2c2ab20SDavid Hildenbrand 634b2c2ab20SDavid Hildenbrand *(unsigned long *)arg += nr_pages; 63575884fb1SKAMEZAWA Hiroyuki return 0; 63675884fb1SKAMEZAWA Hiroyuki } 63775884fb1SKAMEZAWA Hiroyuki 638d9713679SLai Jiangshan /* check which state of node_states will be changed when online memory */ 639d9713679SLai Jiangshan static void node_states_check_changes_online(unsigned long nr_pages, 640d9713679SLai Jiangshan struct zone *zone, struct memory_notify *arg) 641d9713679SLai Jiangshan { 642d9713679SLai Jiangshan int nid = zone_to_nid(zone); 643d9713679SLai Jiangshan 64498fa15f3SAnshuman Khandual arg->status_change_nid = NUMA_NO_NODE; 64598fa15f3SAnshuman Khandual arg->status_change_nid_normal = NUMA_NO_NODE; 64698fa15f3SAnshuman Khandual arg->status_change_nid_high = NUMA_NO_NODE; 6476715ddf9SLai Jiangshan 6486715ddf9SLai Jiangshan if (!node_state(nid, N_MEMORY)) 649d9713679SLai Jiangshan arg->status_change_nid = nid; 6508efe33f4SOscar Salvador if (zone_idx(zone) <= ZONE_NORMAL && !node_state(nid, N_NORMAL_MEMORY)) 6518efe33f4SOscar Salvador arg->status_change_nid_normal = nid; 6528efe33f4SOscar Salvador #ifdef CONFIG_HIGHMEM 653d3ba3ae1SBaoquan He if (zone_idx(zone) <= ZONE_HIGHMEM && !node_state(nid, N_HIGH_MEMORY)) 6548efe33f4SOscar Salvador arg->status_change_nid_high = nid; 6558efe33f4SOscar Salvador #endif 656d9713679SLai Jiangshan } 657d9713679SLai Jiangshan 658d9713679SLai Jiangshan static void node_states_set_node(int node, struct memory_notify *arg) 659d9713679SLai Jiangshan { 660d9713679SLai Jiangshan if (arg->status_change_nid_normal >= 0) 661d9713679SLai Jiangshan node_set_state(node, N_NORMAL_MEMORY); 662d9713679SLai Jiangshan 6636715ddf9SLai Jiangshan if (arg->status_change_nid_high >= 0) 664d9713679SLai Jiangshan node_set_state(node, N_HIGH_MEMORY); 6656715ddf9SLai Jiangshan 66683d83612SOscar Salvador if (arg->status_change_nid >= 0) 6676715ddf9SLai Jiangshan node_set_state(node, N_MEMORY); 668d9713679SLai Jiangshan } 669d9713679SLai Jiangshan 670f1dd2cd1SMichal Hocko static void __meminit resize_zone_range(struct zone *zone, unsigned long start_pfn, 671f1dd2cd1SMichal Hocko unsigned long nr_pages) 672f1dd2cd1SMichal Hocko { 673f1dd2cd1SMichal Hocko unsigned long old_end_pfn = zone_end_pfn(zone); 674f1dd2cd1SMichal Hocko 675f1dd2cd1SMichal Hocko if (zone_is_empty(zone) || start_pfn < zone->zone_start_pfn) 676f1dd2cd1SMichal Hocko zone->zone_start_pfn = start_pfn; 677f1dd2cd1SMichal Hocko 678f1dd2cd1SMichal Hocko zone->spanned_pages = max(start_pfn + nr_pages, old_end_pfn) - zone->zone_start_pfn; 679f1dd2cd1SMichal Hocko } 680f1dd2cd1SMichal Hocko 681f1dd2cd1SMichal Hocko static void __meminit resize_pgdat_range(struct pglist_data *pgdat, unsigned long start_pfn, 682f1dd2cd1SMichal Hocko unsigned long nr_pages) 683f1dd2cd1SMichal Hocko { 684f1dd2cd1SMichal Hocko unsigned long old_end_pfn = pgdat_end_pfn(pgdat); 685f1dd2cd1SMichal Hocko 686f1dd2cd1SMichal Hocko if (!pgdat->node_spanned_pages || start_pfn < pgdat->node_start_pfn) 687f1dd2cd1SMichal Hocko pgdat->node_start_pfn = start_pfn; 688f1dd2cd1SMichal Hocko 689f1dd2cd1SMichal Hocko pgdat->node_spanned_pages = max(start_pfn + nr_pages, old_end_pfn) - pgdat->node_start_pfn; 690f1dd2cd1SMichal Hocko 6913fccb74cSDavid Hildenbrand } 6923fccb74cSDavid Hildenbrand /* 6933fccb74cSDavid Hildenbrand * Associate the pfn range with the given zone, initializing the memmaps 6943fccb74cSDavid Hildenbrand * and resizing the pgdat/zone data to span the added pages. After this 6953fccb74cSDavid Hildenbrand * call, all affected pages are PG_reserved. 6963fccb74cSDavid Hildenbrand */ 697a99583e7SChristoph Hellwig void __ref move_pfn_range_to_zone(struct zone *zone, unsigned long start_pfn, 698a99583e7SChristoph Hellwig unsigned long nr_pages, struct vmem_altmap *altmap) 699f1dd2cd1SMichal Hocko { 700f1dd2cd1SMichal Hocko struct pglist_data *pgdat = zone->zone_pgdat; 701f1dd2cd1SMichal Hocko int nid = pgdat->node_id; 702f1dd2cd1SMichal Hocko unsigned long flags; 703f1dd2cd1SMichal Hocko 704f1dd2cd1SMichal Hocko clear_zone_contiguous(zone); 705f1dd2cd1SMichal Hocko 706f1dd2cd1SMichal Hocko /* TODO Huh pgdat is irqsave while zone is not. It used to be like that before */ 707f1dd2cd1SMichal Hocko pgdat_resize_lock(pgdat, &flags); 708f1dd2cd1SMichal Hocko zone_span_writelock(zone); 709fa004ab7SWei Yang if (zone_is_empty(zone)) 710fa004ab7SWei Yang init_currently_empty_zone(zone, start_pfn, nr_pages); 711f1dd2cd1SMichal Hocko resize_zone_range(zone, start_pfn, nr_pages); 712f1dd2cd1SMichal Hocko zone_span_writeunlock(zone); 713f1dd2cd1SMichal Hocko resize_pgdat_range(pgdat, start_pfn, nr_pages); 714f1dd2cd1SMichal Hocko pgdat_resize_unlock(pgdat, &flags); 715f1dd2cd1SMichal Hocko 716f1dd2cd1SMichal Hocko /* 717f1dd2cd1SMichal Hocko * TODO now we have a visible range of pages which are not associated 718f1dd2cd1SMichal Hocko * with their zone properly. Not nice but set_pfnblock_flags_mask 719f1dd2cd1SMichal Hocko * expects the zone spans the pfn range. All the pages in the range 720f1dd2cd1SMichal Hocko * are reserved so nobody should be touching them so we should be safe 721f1dd2cd1SMichal Hocko */ 722a99583e7SChristoph Hellwig memmap_init_zone(nr_pages, nid, zone_idx(zone), start_pfn, 723a99583e7SChristoph Hellwig MEMMAP_HOTPLUG, altmap); 724f1dd2cd1SMichal Hocko 725f1dd2cd1SMichal Hocko set_zone_contiguous(zone); 726f1dd2cd1SMichal Hocko } 727f1dd2cd1SMichal Hocko 728f1dd2cd1SMichal Hocko /* 729c246a213SMichal Hocko * Returns a default kernel memory zone for the given pfn range. 730c246a213SMichal Hocko * If no kernel zone covers this pfn range it will automatically go 731c246a213SMichal Hocko * to the ZONE_NORMAL. 732c246a213SMichal Hocko */ 733c6f03e29SMichal Hocko static struct zone *default_kernel_zone_for_pfn(int nid, unsigned long start_pfn, 734c246a213SMichal Hocko unsigned long nr_pages) 735c246a213SMichal Hocko { 736c246a213SMichal Hocko struct pglist_data *pgdat = NODE_DATA(nid); 737c246a213SMichal Hocko int zid; 738c246a213SMichal Hocko 739c246a213SMichal Hocko for (zid = 0; zid <= ZONE_NORMAL; zid++) { 740c246a213SMichal Hocko struct zone *zone = &pgdat->node_zones[zid]; 741c246a213SMichal Hocko 742c246a213SMichal Hocko if (zone_intersects(zone, start_pfn, nr_pages)) 743c246a213SMichal Hocko return zone; 744c246a213SMichal Hocko } 745c246a213SMichal Hocko 746c246a213SMichal Hocko return &pgdat->node_zones[ZONE_NORMAL]; 747c246a213SMichal Hocko } 748c246a213SMichal Hocko 749c6f03e29SMichal Hocko static inline struct zone *default_zone_for_pfn(int nid, unsigned long start_pfn, 750c6f03e29SMichal Hocko unsigned long nr_pages) 751e5e68930SMichal Hocko { 752c6f03e29SMichal Hocko struct zone *kernel_zone = default_kernel_zone_for_pfn(nid, start_pfn, 753c6f03e29SMichal Hocko nr_pages); 754c6f03e29SMichal Hocko struct zone *movable_zone = &NODE_DATA(nid)->node_zones[ZONE_MOVABLE]; 755c6f03e29SMichal Hocko bool in_kernel = zone_intersects(kernel_zone, start_pfn, nr_pages); 756c6f03e29SMichal Hocko bool in_movable = zone_intersects(movable_zone, start_pfn, nr_pages); 757e5e68930SMichal Hocko 758e5e68930SMichal Hocko /* 759c6f03e29SMichal Hocko * We inherit the existing zone in a simple case where zones do not 760c6f03e29SMichal Hocko * overlap in the given range 761e5e68930SMichal Hocko */ 762c6f03e29SMichal Hocko if (in_kernel ^ in_movable) 763c6f03e29SMichal Hocko return (in_kernel) ? kernel_zone : movable_zone; 764e5e68930SMichal Hocko 765c6f03e29SMichal Hocko /* 766c6f03e29SMichal Hocko * If the range doesn't belong to any zone or two zones overlap in the 767c6f03e29SMichal Hocko * given range then we use movable zone only if movable_node is 768c6f03e29SMichal Hocko * enabled because we always online to a kernel zone by default. 769c6f03e29SMichal Hocko */ 770c6f03e29SMichal Hocko return movable_node_enabled ? movable_zone : kernel_zone; 7719f123ab5SMichal Hocko } 7729f123ab5SMichal Hocko 773e5e68930SMichal Hocko struct zone * zone_for_pfn_range(int online_type, int nid, unsigned start_pfn, 774e5e68930SMichal Hocko unsigned long nr_pages) 775f1dd2cd1SMichal Hocko { 776c6f03e29SMichal Hocko if (online_type == MMOP_ONLINE_KERNEL) 777c6f03e29SMichal Hocko return default_kernel_zone_for_pfn(nid, start_pfn, nr_pages); 778f1dd2cd1SMichal Hocko 779c6f03e29SMichal Hocko if (online_type == MMOP_ONLINE_MOVABLE) 780c6f03e29SMichal Hocko return &NODE_DATA(nid)->node_zones[ZONE_MOVABLE]; 781f1dd2cd1SMichal Hocko 782c6f03e29SMichal Hocko return default_zone_for_pfn(nid, start_pfn, nr_pages); 783e5e68930SMichal Hocko } 784e5e68930SMichal Hocko 785511c2abaSLai Jiangshan int __ref online_pages(unsigned long pfn, unsigned long nr_pages, int online_type) 78675884fb1SKAMEZAWA Hiroyuki { 787aa47228aSCody P Schafer unsigned long flags; 7883947be19SDave Hansen unsigned long onlined_pages = 0; 7893947be19SDave Hansen struct zone *zone; 7906811378eSYasunori Goto int need_zonelists_rebuild = 0; 7917b78d335SYasunori Goto int nid; 7927b78d335SYasunori Goto int ret; 7937b78d335SYasunori Goto struct memory_notify arg; 794d0dc12e8SPavel Tatashin struct memory_block *mem; 7953947be19SDave Hansen 796381eab4aSDavid Hildenbrand mem_hotplug_begin(); 797381eab4aSDavid Hildenbrand 798d0dc12e8SPavel Tatashin /* 799d0dc12e8SPavel Tatashin * We can't use pfn_to_nid() because nid might be stored in struct page 800d0dc12e8SPavel Tatashin * which is not yet initialized. Instead, we find nid from memory block. 801d0dc12e8SPavel Tatashin */ 802d0dc12e8SPavel Tatashin mem = find_memory_block(__pfn_to_section(pfn)); 803d0dc12e8SPavel Tatashin nid = mem->nid; 80489c02e69SDavid Hildenbrand put_device(&mem->dev); 805d0dc12e8SPavel Tatashin 806f1dd2cd1SMichal Hocko /* associate pfn range with the zone */ 8073fccb74cSDavid Hildenbrand zone = zone_for_pfn_range(online_type, nid, pfn, nr_pages); 8083fccb74cSDavid Hildenbrand move_pfn_range_to_zone(zone, pfn, nr_pages, NULL); 809511c2abaSLai Jiangshan 8107b78d335SYasunori Goto arg.start_pfn = pfn; 8117b78d335SYasunori Goto arg.nr_pages = nr_pages; 812d9713679SLai Jiangshan node_states_check_changes_online(nr_pages, zone, &arg); 8137b78d335SYasunori Goto 8147b78d335SYasunori Goto ret = memory_notify(MEM_GOING_ONLINE, &arg); 8157b78d335SYasunori Goto ret = notifier_to_errno(ret); 816e33e33b4SChen Yucong if (ret) 817e33e33b4SChen Yucong goto failed_addition; 818e33e33b4SChen Yucong 8193947be19SDave Hansen /* 8206811378eSYasunori Goto * If this zone is not populated, then it is not in zonelist. 8216811378eSYasunori Goto * This means the page allocator ignores this zone. 8226811378eSYasunori Goto * So, zonelist must be updated after online. 8236811378eSYasunori Goto */ 8246dcd73d7SWen Congyang if (!populated_zone(zone)) { 8256811378eSYasunori Goto need_zonelists_rebuild = 1; 82672675e13SMichal Hocko setup_zone_pageset(zone); 8276dcd73d7SWen Congyang } 8286811378eSYasunori Goto 829908eedc6SKAMEZAWA Hiroyuki ret = walk_system_ram_range(pfn, nr_pages, &onlined_pages, 83075884fb1SKAMEZAWA Hiroyuki online_pages_range); 831fd8a4221SGeoff Levand if (ret) { 832ca9a46f8SDavid Hildenbrand /* not a single memory resource was applicable */ 8336dcd73d7SWen Congyang if (need_zonelists_rebuild) 8346dcd73d7SWen Congyang zone_pcp_reset(zone); 835e33e33b4SChen Yucong goto failed_addition; 836fd8a4221SGeoff Levand } 837fd8a4221SGeoff Levand 8383947be19SDave Hansen zone->present_pages += onlined_pages; 839aa47228aSCody P Schafer 840aa47228aSCody P Schafer pgdat_resize_lock(zone->zone_pgdat, &flags); 841f2937be5SYasunori Goto zone->zone_pgdat->node_present_pages += onlined_pages; 842aa47228aSCody P Schafer pgdat_resize_unlock(zone->zone_pgdat, &flags); 843aa47228aSCody P Schafer 844e900a918SDan Williams shuffle_zone(zone); 845e900a918SDan Williams 846e888ca35SVlastimil Babka node_states_set_node(nid, &arg); 8471f522509SHaicheng Li if (need_zonelists_rebuild) 84872675e13SMichal Hocko build_all_zonelists(NULL); 8491f522509SHaicheng Li else 850112067f0SShaohua Li zone_pcp_update(zone); 8511f522509SHaicheng Li 8521b79acc9SKOSAKI Motohiro init_per_zone_wmark_min(); 8531b79acc9SKOSAKI Motohiro 854e888ca35SVlastimil Babka kswapd_run(nid); 855698b1b30SVlastimil Babka kcompactd_run(nid); 85661b13993SDave Hansen 8575a4d4361SKAMEZAWA Hiroyuki vm_total_pages = nr_free_pagecache_pages(); 8582f7f24ecSKent Liu 8592d1d43f6SChandra Seetharaman writeback_set_ratelimit(); 8607b78d335SYasunori Goto 8617b78d335SYasunori Goto memory_notify(MEM_ONLINE, &arg); 862381eab4aSDavid Hildenbrand mem_hotplug_done(); 86330467e0bSDavid Rientjes return 0; 864e33e33b4SChen Yucong 865e33e33b4SChen Yucong failed_addition: 866e33e33b4SChen Yucong pr_debug("online_pages [mem %#010llx-%#010llx] failed\n", 867e33e33b4SChen Yucong (unsigned long long) pfn << PAGE_SHIFT, 868e33e33b4SChen Yucong (((unsigned long long) pfn + nr_pages) << PAGE_SHIFT) - 1); 869e33e33b4SChen Yucong memory_notify(MEM_CANCEL_ONLINE, &arg); 870381eab4aSDavid Hildenbrand mem_hotplug_done(); 871e33e33b4SChen Yucong return ret; 8723947be19SDave Hansen } 87353947027SKeith Mannthey #endif /* CONFIG_MEMORY_HOTPLUG_SPARSE */ 874bc02af93SYasunori Goto 8750bd85420STang Chen static void reset_node_present_pages(pg_data_t *pgdat) 8760bd85420STang Chen { 8770bd85420STang Chen struct zone *z; 8780bd85420STang Chen 8790bd85420STang Chen for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++) 8800bd85420STang Chen z->present_pages = 0; 8810bd85420STang Chen 8820bd85420STang Chen pgdat->node_present_pages = 0; 8830bd85420STang Chen } 8840bd85420STang Chen 885e1319331SHidetoshi Seto /* we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG */ 886e1319331SHidetoshi Seto static pg_data_t __ref *hotadd_new_pgdat(int nid, u64 start) 8879af3c2deSYasunori Goto { 8889af3c2deSYasunori Goto struct pglist_data *pgdat; 889c8e861a5SFabian Frederick unsigned long start_pfn = PFN_DOWN(start); 8909af3c2deSYasunori Goto 891a1e565aaSTang Chen pgdat = NODE_DATA(nid); 892a1e565aaSTang Chen if (!pgdat) { 8939af3c2deSYasunori Goto pgdat = arch_alloc_nodedata(nid); 8949af3c2deSYasunori Goto if (!pgdat) 8959af3c2deSYasunori Goto return NULL; 8969af3c2deSYasunori Goto 89733fce011SWei Yang pgdat->per_cpu_nodestats = 89833fce011SWei Yang alloc_percpu(struct per_cpu_nodestat); 8999af3c2deSYasunori Goto arch_refresh_nodedata(nid, pgdat); 900b0dc3a34SGu Zheng } else { 90133fce011SWei Yang int cpu; 902e716f2ebSMel Gorman /* 903e716f2ebSMel Gorman * Reset the nr_zones, order and classzone_idx before reuse. 904e716f2ebSMel Gorman * Note that kswapd will init kswapd_classzone_idx properly 905e716f2ebSMel Gorman * when it starts in the near future. 906e716f2ebSMel Gorman */ 907b0dc3a34SGu Zheng pgdat->nr_zones = 0; 90838087d9bSMel Gorman pgdat->kswapd_order = 0; 90938087d9bSMel Gorman pgdat->kswapd_classzone_idx = 0; 91033fce011SWei Yang for_each_online_cpu(cpu) { 91133fce011SWei Yang struct per_cpu_nodestat *p; 91233fce011SWei Yang 91333fce011SWei Yang p = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu); 91433fce011SWei Yang memset(p, 0, sizeof(*p)); 91533fce011SWei Yang } 916a1e565aaSTang Chen } 9179af3c2deSYasunori Goto 9189af3c2deSYasunori Goto /* we can use NODE_DATA(nid) from here */ 9199af3c2deSYasunori Goto 92003e85f9dSOscar Salvador pgdat->node_id = nid; 92103e85f9dSOscar Salvador pgdat->node_start_pfn = start_pfn; 92203e85f9dSOscar Salvador 9239af3c2deSYasunori Goto /* init node's zones as empty zones, we don't have any present pages.*/ 92403e85f9dSOscar Salvador free_area_init_core_hotplug(nid); 9259af3c2deSYasunori Goto 926959ecc48SKAMEZAWA Hiroyuki /* 927959ecc48SKAMEZAWA Hiroyuki * The node we allocated has no zone fallback lists. For avoiding 928959ecc48SKAMEZAWA Hiroyuki * to access not-initialized zonelist, build here. 929959ecc48SKAMEZAWA Hiroyuki */ 93072675e13SMichal Hocko build_all_zonelists(pgdat); 931959ecc48SKAMEZAWA Hiroyuki 932f784a3f1STang Chen /* 9330bd85420STang Chen * When memory is hot-added, all the memory is in offline state. So 9340bd85420STang Chen * clear all zones' present_pages because they will be updated in 9350bd85420STang Chen * online_pages() and offline_pages(). 9360bd85420STang Chen */ 93703e85f9dSOscar Salvador reset_node_managed_pages(pgdat); 9380bd85420STang Chen reset_node_present_pages(pgdat); 9390bd85420STang Chen 9409af3c2deSYasunori Goto return pgdat; 9419af3c2deSYasunori Goto } 9429af3c2deSYasunori Goto 943b9ff0360SOscar Salvador static void rollback_node_hotadd(int nid) 9449af3c2deSYasunori Goto { 945b9ff0360SOscar Salvador pg_data_t *pgdat = NODE_DATA(nid); 946b9ff0360SOscar Salvador 9479af3c2deSYasunori Goto arch_refresh_nodedata(nid, NULL); 9485830169fSReza Arbab free_percpu(pgdat->per_cpu_nodestats); 9499af3c2deSYasunori Goto arch_free_nodedata(pgdat); 9509af3c2deSYasunori Goto } 9519af3c2deSYasunori Goto 9520a547039SKAMEZAWA Hiroyuki 95301b0f197SToshi Kani /** 95401b0f197SToshi Kani * try_online_node - online a node if offlined 955e8b098fcSMike Rapoport * @nid: the node ID 956b9ff0360SOscar Salvador * @start: start addr of the node 957b9ff0360SOscar Salvador * @set_node_online: Whether we want to online the node 958cf23422bSminskey guo * called by cpu_up() to online a node without onlined memory. 959b9ff0360SOscar Salvador * 960b9ff0360SOscar Salvador * Returns: 961b9ff0360SOscar Salvador * 1 -> a new node has been allocated 962b9ff0360SOscar Salvador * 0 -> the node is already online 963b9ff0360SOscar Salvador * -ENOMEM -> the node could not be allocated 964cf23422bSminskey guo */ 965b9ff0360SOscar Salvador static int __try_online_node(int nid, u64 start, bool set_node_online) 966cf23422bSminskey guo { 967cf23422bSminskey guo pg_data_t *pgdat; 968b9ff0360SOscar Salvador int ret = 1; 969cf23422bSminskey guo 97001b0f197SToshi Kani if (node_online(nid)) 97101b0f197SToshi Kani return 0; 97201b0f197SToshi Kani 973b9ff0360SOscar Salvador pgdat = hotadd_new_pgdat(nid, start); 9747553e8f2SDavid Rientjes if (!pgdat) { 97501b0f197SToshi Kani pr_err("Cannot online node %d due to NULL pgdat\n", nid); 976cf23422bSminskey guo ret = -ENOMEM; 977cf23422bSminskey guo goto out; 978cf23422bSminskey guo } 979b9ff0360SOscar Salvador 980b9ff0360SOscar Salvador if (set_node_online) { 981cf23422bSminskey guo node_set_online(nid); 982cf23422bSminskey guo ret = register_one_node(nid); 983cf23422bSminskey guo BUG_ON(ret); 984b9ff0360SOscar Salvador } 985cf23422bSminskey guo out: 986b9ff0360SOscar Salvador return ret; 987b9ff0360SOscar Salvador } 988b9ff0360SOscar Salvador 989b9ff0360SOscar Salvador /* 990b9ff0360SOscar Salvador * Users of this function always want to online/register the node 991b9ff0360SOscar Salvador */ 992b9ff0360SOscar Salvador int try_online_node(int nid) 993b9ff0360SOscar Salvador { 994b9ff0360SOscar Salvador int ret; 995b9ff0360SOscar Salvador 996b9ff0360SOscar Salvador mem_hotplug_begin(); 997b9ff0360SOscar Salvador ret = __try_online_node(nid, 0, true); 998bfc8c901SVladimir Davydov mem_hotplug_done(); 999cf23422bSminskey guo return ret; 1000cf23422bSminskey guo } 1001cf23422bSminskey guo 100227356f54SToshi Kani static int check_hotplug_memory_range(u64 start, u64 size) 100327356f54SToshi Kani { 1004ba325585SPavel Tatashin /* memory range must be block size aligned */ 1005cec3ebd0SDavid Hildenbrand if (!size || !IS_ALIGNED(start, memory_block_size_bytes()) || 1006cec3ebd0SDavid Hildenbrand !IS_ALIGNED(size, memory_block_size_bytes())) { 1007ba325585SPavel Tatashin pr_err("Block size [%#lx] unaligned hotplug range: start %#llx, size %#llx", 1008cec3ebd0SDavid Hildenbrand memory_block_size_bytes(), start, size); 100927356f54SToshi Kani return -EINVAL; 101027356f54SToshi Kani } 101127356f54SToshi Kani 101227356f54SToshi Kani return 0; 101327356f54SToshi Kani } 101427356f54SToshi Kani 101531bc3858SVitaly Kuznetsov static int online_memory_block(struct memory_block *mem, void *arg) 101631bc3858SVitaly Kuznetsov { 1017dc18d706SNathan Fontenot return device_online(&mem->dev); 101831bc3858SVitaly Kuznetsov } 101931bc3858SVitaly Kuznetsov 10208df1d0e4SDavid Hildenbrand /* 10218df1d0e4SDavid Hildenbrand * NOTE: The caller must call lock_device_hotplug() to serialize hotplug 10228df1d0e4SDavid Hildenbrand * and online/offline operations (triggered e.g. by sysfs). 10238df1d0e4SDavid Hildenbrand * 10248df1d0e4SDavid Hildenbrand * we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG 10258df1d0e4SDavid Hildenbrand */ 1026f29d8e9cSDavid Hildenbrand int __ref add_memory_resource(int nid, struct resource *res) 1027bc02af93SYasunori Goto { 102805f800a0SDavid Hildenbrand struct mhp_restrictions restrictions = {}; 102962cedb9fSDavid Vrabel u64 start, size; 1030b9ff0360SOscar Salvador bool new_node = false; 1031bc02af93SYasunori Goto int ret; 1032bc02af93SYasunori Goto 103362cedb9fSDavid Vrabel start = res->start; 103462cedb9fSDavid Vrabel size = resource_size(res); 103562cedb9fSDavid Vrabel 103627356f54SToshi Kani ret = check_hotplug_memory_range(start, size); 103727356f54SToshi Kani if (ret) 103827356f54SToshi Kani return ret; 103927356f54SToshi Kani 1040bfc8c901SVladimir Davydov mem_hotplug_begin(); 1041ac13c462SNathan Zimmer 10427f36e3e5STang Chen /* 10437f36e3e5STang Chen * Add new range to memblock so that when hotadd_new_pgdat() is called 10447f36e3e5STang Chen * to allocate new pgdat, get_pfn_range_for_nid() will be able to find 10457f36e3e5STang Chen * this new range and calculate total pages correctly. The range will 10467f36e3e5STang Chen * be removed at hot-remove time. 10477f36e3e5STang Chen */ 10487f36e3e5STang Chen memblock_add_node(start, size, nid); 10497f36e3e5STang Chen 1050b9ff0360SOscar Salvador ret = __try_online_node(nid, start, false); 1051b9ff0360SOscar Salvador if (ret < 0) 105241b9e2d7SWen Congyang goto error; 1053b9ff0360SOscar Salvador new_node = ret; 10549af3c2deSYasunori Goto 1055bc02af93SYasunori Goto /* call arch's memory hotadd */ 1056940519f0SMichal Hocko ret = arch_add_memory(nid, start, size, &restrictions); 10579af3c2deSYasunori Goto if (ret < 0) 10589af3c2deSYasunori Goto goto error; 10599af3c2deSYasunori Goto 1060db051a0dSDavid Hildenbrand /* create memory block devices after memory was added */ 1061db051a0dSDavid Hildenbrand ret = create_memory_block_devices(start, size); 1062db051a0dSDavid Hildenbrand if (ret) { 1063db051a0dSDavid Hildenbrand arch_remove_memory(nid, start, size, NULL); 1064db051a0dSDavid Hildenbrand goto error; 1065db051a0dSDavid Hildenbrand } 1066db051a0dSDavid Hildenbrand 1067a1e565aaSTang Chen if (new_node) { 1068d5b6f6a3SOscar Salvador /* If sysfs file of new node can't be created, cpu on the node 10690fc44159SYasunori Goto * can't be hot-added. There is no rollback way now. 10700fc44159SYasunori Goto * So, check by BUG_ON() to catch it reluctantly.. 1071d5b6f6a3SOscar Salvador * We online node here. We can't roll back from here. 10720fc44159SYasunori Goto */ 1073d5b6f6a3SOscar Salvador node_set_online(nid); 1074d5b6f6a3SOscar Salvador ret = __register_one_node(nid); 10750fc44159SYasunori Goto BUG_ON(ret); 10760fc44159SYasunori Goto } 10770fc44159SYasunori Goto 1078d5b6f6a3SOscar Salvador /* link memory sections under this node.*/ 10794fbce633SOscar Salvador ret = link_mem_sections(nid, PFN_DOWN(start), PFN_UP(start + size - 1)); 1080d5b6f6a3SOscar Salvador BUG_ON(ret); 1081d5b6f6a3SOscar Salvador 1082d96ae530Sakpm@linux-foundation.org /* create new memmap entry */ 1083d96ae530Sakpm@linux-foundation.org firmware_map_add_hotplug(start, start + size, "System RAM"); 1084d96ae530Sakpm@linux-foundation.org 1085381eab4aSDavid Hildenbrand /* device_online() will take the lock when calling online_pages() */ 1086381eab4aSDavid Hildenbrand mem_hotplug_done(); 1087381eab4aSDavid Hildenbrand 108831bc3858SVitaly Kuznetsov /* online pages if requested */ 1089f29d8e9cSDavid Hildenbrand if (memhp_auto_online) 1090fbcf73ceSDavid Hildenbrand walk_memory_blocks(start, size, NULL, online_memory_block); 109131bc3858SVitaly Kuznetsov 1092381eab4aSDavid Hildenbrand return ret; 10939af3c2deSYasunori Goto error: 10949af3c2deSYasunori Goto /* rollback pgdat allocation and others */ 1095b9ff0360SOscar Salvador if (new_node) 1096b9ff0360SOscar Salvador rollback_node_hotadd(nid); 10977f36e3e5STang Chen memblock_remove(start, size); 1098bfc8c901SVladimir Davydov mem_hotplug_done(); 1099bc02af93SYasunori Goto return ret; 1100bc02af93SYasunori Goto } 110162cedb9fSDavid Vrabel 11028df1d0e4SDavid Hildenbrand /* requires device_hotplug_lock, see add_memory_resource() */ 11038df1d0e4SDavid Hildenbrand int __ref __add_memory(int nid, u64 start, u64 size) 110462cedb9fSDavid Vrabel { 110562cedb9fSDavid Vrabel struct resource *res; 110662cedb9fSDavid Vrabel int ret; 110762cedb9fSDavid Vrabel 110862cedb9fSDavid Vrabel res = register_memory_resource(start, size); 11096f754ba4SVitaly Kuznetsov if (IS_ERR(res)) 11106f754ba4SVitaly Kuznetsov return PTR_ERR(res); 111162cedb9fSDavid Vrabel 1112f29d8e9cSDavid Hildenbrand ret = add_memory_resource(nid, res); 111362cedb9fSDavid Vrabel if (ret < 0) 111462cedb9fSDavid Vrabel release_memory_resource(res); 111562cedb9fSDavid Vrabel return ret; 111662cedb9fSDavid Vrabel } 11178df1d0e4SDavid Hildenbrand 11188df1d0e4SDavid Hildenbrand int add_memory(int nid, u64 start, u64 size) 11198df1d0e4SDavid Hildenbrand { 11208df1d0e4SDavid Hildenbrand int rc; 11218df1d0e4SDavid Hildenbrand 11228df1d0e4SDavid Hildenbrand lock_device_hotplug(); 11238df1d0e4SDavid Hildenbrand rc = __add_memory(nid, start, size); 11248df1d0e4SDavid Hildenbrand unlock_device_hotplug(); 11258df1d0e4SDavid Hildenbrand 11268df1d0e4SDavid Hildenbrand return rc; 11278df1d0e4SDavid Hildenbrand } 1128bc02af93SYasunori Goto EXPORT_SYMBOL_GPL(add_memory); 11290c0e6195SKAMEZAWA Hiroyuki 11300c0e6195SKAMEZAWA Hiroyuki #ifdef CONFIG_MEMORY_HOTREMOVE 11310c0e6195SKAMEZAWA Hiroyuki /* 11325c755e9fSBadari Pulavarty * A free page on the buddy free lists (not the per-cpu lists) has PageBuddy 11335c755e9fSBadari Pulavarty * set and the size of the free page is given by page_order(). Using this, 11345c755e9fSBadari Pulavarty * the function determines if the pageblock contains only free pages. 11355c755e9fSBadari Pulavarty * Due to buddy contraints, a free page at least the size of a pageblock will 11365c755e9fSBadari Pulavarty * be located at the start of the pageblock 11375c755e9fSBadari Pulavarty */ 11385c755e9fSBadari Pulavarty static inline int pageblock_free(struct page *page) 11395c755e9fSBadari Pulavarty { 11405c755e9fSBadari Pulavarty return PageBuddy(page) && page_order(page) >= pageblock_order; 11415c755e9fSBadari Pulavarty } 11425c755e9fSBadari Pulavarty 1143891cb2a7SMichal Hocko /* Return the pfn of the start of the next active pageblock after a given pfn */ 1144891cb2a7SMichal Hocko static unsigned long next_active_pageblock(unsigned long pfn) 11455c755e9fSBadari Pulavarty { 1146891cb2a7SMichal Hocko struct page *page = pfn_to_page(pfn); 1147891cb2a7SMichal Hocko 11485c755e9fSBadari Pulavarty /* Ensure the starting page is pageblock-aligned */ 1149891cb2a7SMichal Hocko BUG_ON(pfn & (pageblock_nr_pages - 1)); 11505c755e9fSBadari Pulavarty 11515c755e9fSBadari Pulavarty /* If the entire pageblock is free, move to the end of free page */ 11520dcc48c1SKAMEZAWA Hiroyuki if (pageblock_free(page)) { 11530dcc48c1SKAMEZAWA Hiroyuki int order; 11540dcc48c1SKAMEZAWA Hiroyuki /* be careful. we don't have locks, page_order can be changed.*/ 11550dcc48c1SKAMEZAWA Hiroyuki order = page_order(page); 11560dcc48c1SKAMEZAWA Hiroyuki if ((order < MAX_ORDER) && (order >= pageblock_order)) 1157891cb2a7SMichal Hocko return pfn + (1 << order); 11580dcc48c1SKAMEZAWA Hiroyuki } 11595c755e9fSBadari Pulavarty 1160891cb2a7SMichal Hocko return pfn + pageblock_nr_pages; 11615c755e9fSBadari Pulavarty } 11625c755e9fSBadari Pulavarty 1163891cb2a7SMichal Hocko static bool is_pageblock_removable_nolock(unsigned long pfn) 1164fb52bbaeSMathieu Malaterre { 1165891cb2a7SMichal Hocko struct page *page = pfn_to_page(pfn); 1166fb52bbaeSMathieu Malaterre struct zone *zone; 1167fb52bbaeSMathieu Malaterre 1168fb52bbaeSMathieu Malaterre /* 1169fb52bbaeSMathieu Malaterre * We have to be careful here because we are iterating over memory 1170fb52bbaeSMathieu Malaterre * sections which are not zone aware so we might end up outside of 1171fb52bbaeSMathieu Malaterre * the zone but still within the section. 1172fb52bbaeSMathieu Malaterre * We have to take care about the node as well. If the node is offline 1173fb52bbaeSMathieu Malaterre * its NODE_DATA will be NULL - see page_zone. 1174fb52bbaeSMathieu Malaterre */ 1175fb52bbaeSMathieu Malaterre if (!node_online(page_to_nid(page))) 1176fb52bbaeSMathieu Malaterre return false; 1177fb52bbaeSMathieu Malaterre 1178fb52bbaeSMathieu Malaterre zone = page_zone(page); 1179fb52bbaeSMathieu Malaterre pfn = page_to_pfn(page); 1180fb52bbaeSMathieu Malaterre if (!zone_spans_pfn(zone, pfn)) 1181fb52bbaeSMathieu Malaterre return false; 1182fb52bbaeSMathieu Malaterre 1183d381c547SMichal Hocko return !has_unmovable_pages(zone, page, 0, MIGRATE_MOVABLE, SKIP_HWPOISON); 1184fb52bbaeSMathieu Malaterre } 1185fb52bbaeSMathieu Malaterre 11865c755e9fSBadari Pulavarty /* Checks if this range of memory is likely to be hot-removable. */ 1187c98940f6SYaowei Bai bool is_mem_section_removable(unsigned long start_pfn, unsigned long nr_pages) 11885c755e9fSBadari Pulavarty { 1189891cb2a7SMichal Hocko unsigned long end_pfn, pfn; 1190891cb2a7SMichal Hocko 1191891cb2a7SMichal Hocko end_pfn = min(start_pfn + nr_pages, 1192891cb2a7SMichal Hocko zone_end_pfn(page_zone(pfn_to_page(start_pfn)))); 11935c755e9fSBadari Pulavarty 11945c755e9fSBadari Pulavarty /* Check the starting page of each pageblock within the range */ 1195891cb2a7SMichal Hocko for (pfn = start_pfn; pfn < end_pfn; pfn = next_active_pageblock(pfn)) { 1196891cb2a7SMichal Hocko if (!is_pageblock_removable_nolock(pfn)) 1197c98940f6SYaowei Bai return false; 119849ac8255SKAMEZAWA Hiroyuki cond_resched(); 11995c755e9fSBadari Pulavarty } 12005c755e9fSBadari Pulavarty 12015c755e9fSBadari Pulavarty /* All pageblocks in the memory block are likely to be hot-removable */ 1202c98940f6SYaowei Bai return true; 12035c755e9fSBadari Pulavarty } 12045c755e9fSBadari Pulavarty 12055c755e9fSBadari Pulavarty /* 1206deb88a2aSToshi Kani * Confirm all pages in a range [start, end) belong to the same zone. 1207a96dfddbSToshi Kani * When true, return its valid [start, end). 12080c0e6195SKAMEZAWA Hiroyuki */ 1209a96dfddbSToshi Kani int test_pages_in_a_zone(unsigned long start_pfn, unsigned long end_pfn, 1210a96dfddbSToshi Kani unsigned long *valid_start, unsigned long *valid_end) 12110c0e6195SKAMEZAWA Hiroyuki { 12125f0f2887SAndrew Banman unsigned long pfn, sec_end_pfn; 1213a96dfddbSToshi Kani unsigned long start, end; 12140c0e6195SKAMEZAWA Hiroyuki struct zone *zone = NULL; 12150c0e6195SKAMEZAWA Hiroyuki struct page *page; 12160c0e6195SKAMEZAWA Hiroyuki int i; 1217deb88a2aSToshi Kani for (pfn = start_pfn, sec_end_pfn = SECTION_ALIGN_UP(start_pfn + 1); 12180c0e6195SKAMEZAWA Hiroyuki pfn < end_pfn; 1219deb88a2aSToshi Kani pfn = sec_end_pfn, sec_end_pfn += PAGES_PER_SECTION) { 12205f0f2887SAndrew Banman /* Make sure the memory section is present first */ 12215f0f2887SAndrew Banman if (!present_section_nr(pfn_to_section_nr(pfn))) 12225f0f2887SAndrew Banman continue; 12235f0f2887SAndrew Banman for (; pfn < sec_end_pfn && pfn < end_pfn; 12240c0e6195SKAMEZAWA Hiroyuki pfn += MAX_ORDER_NR_PAGES) { 12250c0e6195SKAMEZAWA Hiroyuki i = 0; 12260c0e6195SKAMEZAWA Hiroyuki /* This is just a CONFIG_HOLES_IN_ZONE check.*/ 12275f0f2887SAndrew Banman while ((i < MAX_ORDER_NR_PAGES) && 12285f0f2887SAndrew Banman !pfn_valid_within(pfn + i)) 12290c0e6195SKAMEZAWA Hiroyuki i++; 1230d6d8c8a4Szhong jiang if (i == MAX_ORDER_NR_PAGES || pfn + i >= end_pfn) 12310c0e6195SKAMEZAWA Hiroyuki continue; 123224feb47cSMikhail Zaslonko /* Check if we got outside of the zone */ 123324feb47cSMikhail Zaslonko if (zone && !zone_spans_pfn(zone, pfn + i)) 123424feb47cSMikhail Zaslonko return 0; 12350c0e6195SKAMEZAWA Hiroyuki page = pfn_to_page(pfn + i); 12360c0e6195SKAMEZAWA Hiroyuki if (zone && page_zone(page) != zone) 12370c0e6195SKAMEZAWA Hiroyuki return 0; 1238a96dfddbSToshi Kani if (!zone) 1239a96dfddbSToshi Kani start = pfn + i; 12400c0e6195SKAMEZAWA Hiroyuki zone = page_zone(page); 1241a96dfddbSToshi Kani end = pfn + MAX_ORDER_NR_PAGES; 12420c0e6195SKAMEZAWA Hiroyuki } 12435f0f2887SAndrew Banman } 1244deb88a2aSToshi Kani 1245a96dfddbSToshi Kani if (zone) { 1246a96dfddbSToshi Kani *valid_start = start; 1247d6d8c8a4Szhong jiang *valid_end = min(end, end_pfn); 12480c0e6195SKAMEZAWA Hiroyuki return 1; 1249a96dfddbSToshi Kani } else { 1250deb88a2aSToshi Kani return 0; 12510c0e6195SKAMEZAWA Hiroyuki } 1252a96dfddbSToshi Kani } 12530c0e6195SKAMEZAWA Hiroyuki 12540c0e6195SKAMEZAWA Hiroyuki /* 12550efadf48SYisheng Xie * Scan pfn range [start,end) to find movable/migratable pages (LRU pages, 12560efadf48SYisheng Xie * non-lru movable pages and hugepages). We scan pfn because it's much 12570efadf48SYisheng Xie * easier than scanning over linked list. This function returns the pfn 12580efadf48SYisheng Xie * of the first found movable page if it's found, otherwise 0. 12590c0e6195SKAMEZAWA Hiroyuki */ 1260c8721bbbSNaoya Horiguchi static unsigned long scan_movable_pages(unsigned long start, unsigned long end) 12610c0e6195SKAMEZAWA Hiroyuki { 12620c0e6195SKAMEZAWA Hiroyuki unsigned long pfn; 1263eeb0efd0SOscar Salvador 12640c0e6195SKAMEZAWA Hiroyuki for (pfn = start; pfn < end; pfn++) { 1265eeb0efd0SOscar Salvador struct page *page, *head; 1266eeb0efd0SOscar Salvador unsigned long skip; 1267eeb0efd0SOscar Salvador 1268eeb0efd0SOscar Salvador if (!pfn_valid(pfn)) 1269eeb0efd0SOscar Salvador continue; 12700c0e6195SKAMEZAWA Hiroyuki page = pfn_to_page(pfn); 12710c0e6195SKAMEZAWA Hiroyuki if (PageLRU(page)) 12720c0e6195SKAMEZAWA Hiroyuki return pfn; 12730efadf48SYisheng Xie if (__PageMovable(page)) 12740efadf48SYisheng Xie return pfn; 1275eeb0efd0SOscar Salvador 1276eeb0efd0SOscar Salvador if (!PageHuge(page)) 1277eeb0efd0SOscar Salvador continue; 1278eeb0efd0SOscar Salvador head = compound_head(page); 127939186cbeSOscar Salvador if (page_huge_active(head)) 1280c8721bbbSNaoya Horiguchi return pfn; 1281d8c6546bSMatthew Wilcox (Oracle) skip = compound_nr(head) - (page - head); 1282eeb0efd0SOscar Salvador pfn += skip - 1; 12830c0e6195SKAMEZAWA Hiroyuki } 12840c0e6195SKAMEZAWA Hiroyuki return 0; 12850c0e6195SKAMEZAWA Hiroyuki } 12860c0e6195SKAMEZAWA Hiroyuki 1287666feb21SMichal Hocko static struct page *new_node_page(struct page *page, unsigned long private) 1288394e31d2SXishi Qiu { 1289394e31d2SXishi Qiu int nid = page_to_nid(page); 1290231e97e2SLi Zhong nodemask_t nmask = node_states[N_MEMORY]; 12917f252f27SMichal Hocko 12927f252f27SMichal Hocko /* 12937f252f27SMichal Hocko * try to allocate from a different node but reuse this node if there 12947f252f27SMichal Hocko * are no other online nodes to be used (e.g. we are offlining a part 12957f252f27SMichal Hocko * of the only existing node) 12967f252f27SMichal Hocko */ 12977f252f27SMichal Hocko node_clear(nid, nmask); 12987f252f27SMichal Hocko if (nodes_empty(nmask)) 12997f252f27SMichal Hocko node_set(nid, nmask); 1300394e31d2SXishi Qiu 13018b913238SMichal Hocko return new_page_nodemask(page, nid, &nmask); 1302394e31d2SXishi Qiu } 1303394e31d2SXishi Qiu 13040c0e6195SKAMEZAWA Hiroyuki static int 13050c0e6195SKAMEZAWA Hiroyuki do_migrate_range(unsigned long start_pfn, unsigned long end_pfn) 13060c0e6195SKAMEZAWA Hiroyuki { 13070c0e6195SKAMEZAWA Hiroyuki unsigned long pfn; 13080c0e6195SKAMEZAWA Hiroyuki struct page *page; 13090c0e6195SKAMEZAWA Hiroyuki int ret = 0; 13100c0e6195SKAMEZAWA Hiroyuki LIST_HEAD(source); 13110c0e6195SKAMEZAWA Hiroyuki 1312a85009c3SMichal Hocko for (pfn = start_pfn; pfn < end_pfn; pfn++) { 13130c0e6195SKAMEZAWA Hiroyuki if (!pfn_valid(pfn)) 13140c0e6195SKAMEZAWA Hiroyuki continue; 13150c0e6195SKAMEZAWA Hiroyuki page = pfn_to_page(pfn); 1316c8721bbbSNaoya Horiguchi 1317c8721bbbSNaoya Horiguchi if (PageHuge(page)) { 1318c8721bbbSNaoya Horiguchi struct page *head = compound_head(page); 1319d8c6546bSMatthew Wilcox (Oracle) pfn = page_to_pfn(head) + compound_nr(head) - 1; 1320daf3538aSOscar Salvador isolate_huge_page(head, &source); 1321c8721bbbSNaoya Horiguchi continue; 132294723aafSMichal Hocko } else if (PageTransHuge(page)) 13238135d892SNaoya Horiguchi pfn = page_to_pfn(compound_head(page)) 13248135d892SNaoya Horiguchi + hpage_nr_pages(page) - 1; 1325c8721bbbSNaoya Horiguchi 1326b15c8726SMichal Hocko /* 1327b15c8726SMichal Hocko * HWPoison pages have elevated reference counts so the migration would 1328b15c8726SMichal Hocko * fail on them. It also doesn't make any sense to migrate them in the 1329b15c8726SMichal Hocko * first place. Still try to unmap such a page in case it is still mapped 1330b15c8726SMichal Hocko * (e.g. current hwpoison implementation doesn't unmap KSM pages but keep 1331b15c8726SMichal Hocko * the unmap as the catch all safety net). 1332b15c8726SMichal Hocko */ 1333b15c8726SMichal Hocko if (PageHWPoison(page)) { 1334b15c8726SMichal Hocko if (WARN_ON(PageLRU(page))) 1335b15c8726SMichal Hocko isolate_lru_page(page); 1336b15c8726SMichal Hocko if (page_mapped(page)) 1337b15c8726SMichal Hocko try_to_unmap(page, TTU_IGNORE_MLOCK | TTU_IGNORE_ACCESS); 1338b15c8726SMichal Hocko continue; 1339b15c8726SMichal Hocko } 1340b15c8726SMichal Hocko 1341700c2a46SKonstantin Khlebnikov if (!get_page_unless_zero(page)) 13420c0e6195SKAMEZAWA Hiroyuki continue; 13430c0e6195SKAMEZAWA Hiroyuki /* 13440efadf48SYisheng Xie * We can skip free pages. And we can deal with pages on 13450efadf48SYisheng Xie * LRU and non-lru movable pages. 13460c0e6195SKAMEZAWA Hiroyuki */ 13470efadf48SYisheng Xie if (PageLRU(page)) 134862695a84SNick Piggin ret = isolate_lru_page(page); 13490efadf48SYisheng Xie else 13500efadf48SYisheng Xie ret = isolate_movable_page(page, ISOLATE_UNEVICTABLE); 13510c0e6195SKAMEZAWA Hiroyuki if (!ret) { /* Success */ 135262695a84SNick Piggin list_add_tail(&page->lru, &source); 13530efadf48SYisheng Xie if (!__PageMovable(page)) 1354599d0c95SMel Gorman inc_node_page_state(page, NR_ISOLATED_ANON + 13556d9c285aSKOSAKI Motohiro page_is_file_cache(page)); 13566d9c285aSKOSAKI Motohiro 13570c0e6195SKAMEZAWA Hiroyuki } else { 13582932c8b0SMichal Hocko pr_warn("failed to isolate pfn %lx\n", pfn); 13590efadf48SYisheng Xie dump_page(page, "isolation failed"); 13601723058eSOscar Salvador } 1361700c2a46SKonstantin Khlebnikov put_page(page); 13620c0e6195SKAMEZAWA Hiroyuki } 1363f3ab2636SBob Liu if (!list_empty(&source)) { 1364394e31d2SXishi Qiu /* Allocate a new page from the nearest neighbor node */ 1365394e31d2SXishi Qiu ret = migrate_pages(&source, new_node_page, NULL, 0, 13669c620e2bSHugh Dickins MIGRATE_SYNC, MR_MEMORY_HOTPLUG); 13672932c8b0SMichal Hocko if (ret) { 13682932c8b0SMichal Hocko list_for_each_entry(page, &source, lru) { 13692932c8b0SMichal Hocko pr_warn("migrating pfn %lx failed ret:%d ", 13702932c8b0SMichal Hocko page_to_pfn(page), ret); 13712932c8b0SMichal Hocko dump_page(page, "migration failure"); 13722932c8b0SMichal Hocko } 1373c8721bbbSNaoya Horiguchi putback_movable_pages(&source); 1374f3ab2636SBob Liu } 13752932c8b0SMichal Hocko } 13761723058eSOscar Salvador 13770c0e6195SKAMEZAWA Hiroyuki return ret; 13780c0e6195SKAMEZAWA Hiroyuki } 13790c0e6195SKAMEZAWA Hiroyuki 13800c0e6195SKAMEZAWA Hiroyuki /* 13810c0e6195SKAMEZAWA Hiroyuki * remove from free_area[] and mark all as Reserved. 13820c0e6195SKAMEZAWA Hiroyuki */ 13830c0e6195SKAMEZAWA Hiroyuki static int 13840c0e6195SKAMEZAWA Hiroyuki offline_isolated_pages_cb(unsigned long start, unsigned long nr_pages, 13850c0e6195SKAMEZAWA Hiroyuki void *data) 13860c0e6195SKAMEZAWA Hiroyuki { 13875557c766SMichal Hocko unsigned long *offlined_pages = (unsigned long *)data; 13880c0e6195SKAMEZAWA Hiroyuki 13895557c766SMichal Hocko *offlined_pages += __offline_isolated_pages(start, start + nr_pages); 13905557c766SMichal Hocko return 0; 13910c0e6195SKAMEZAWA Hiroyuki } 13920c0e6195SKAMEZAWA Hiroyuki 13930c0e6195SKAMEZAWA Hiroyuki /* 13940c0e6195SKAMEZAWA Hiroyuki * Check all pages in range, recoreded as memory resource, are isolated. 13950c0e6195SKAMEZAWA Hiroyuki */ 13960c0e6195SKAMEZAWA Hiroyuki static int 13970c0e6195SKAMEZAWA Hiroyuki check_pages_isolated_cb(unsigned long start_pfn, unsigned long nr_pages, 13980c0e6195SKAMEZAWA Hiroyuki void *data) 13990c0e6195SKAMEZAWA Hiroyuki { 14005557c766SMichal Hocko return test_pages_isolated(start_pfn, start_pfn + nr_pages, true); 14010c0e6195SKAMEZAWA Hiroyuki } 14020c0e6195SKAMEZAWA Hiroyuki 1403c5320926STang Chen static int __init cmdline_parse_movable_node(char *p) 1404c5320926STang Chen { 14054932381eSMichal Hocko #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 140655ac590cSTang Chen movable_node_enabled = true; 14074932381eSMichal Hocko #else 14084932381eSMichal Hocko pr_warn("movable_node parameter depends on CONFIG_HAVE_MEMBLOCK_NODE_MAP to work properly\n"); 14094932381eSMichal Hocko #endif 1410c5320926STang Chen return 0; 1411c5320926STang Chen } 1412c5320926STang Chen early_param("movable_node", cmdline_parse_movable_node); 1413c5320926STang Chen 1414d9713679SLai Jiangshan /* check which state of node_states will be changed when offline memory */ 1415d9713679SLai Jiangshan static void node_states_check_changes_offline(unsigned long nr_pages, 1416d9713679SLai Jiangshan struct zone *zone, struct memory_notify *arg) 1417d9713679SLai Jiangshan { 1418d9713679SLai Jiangshan struct pglist_data *pgdat = zone->zone_pgdat; 1419d9713679SLai Jiangshan unsigned long present_pages = 0; 142086b27beaSOscar Salvador enum zone_type zt; 1421d9713679SLai Jiangshan 142298fa15f3SAnshuman Khandual arg->status_change_nid = NUMA_NO_NODE; 142398fa15f3SAnshuman Khandual arg->status_change_nid_normal = NUMA_NO_NODE; 142498fa15f3SAnshuman Khandual arg->status_change_nid_high = NUMA_NO_NODE; 142586b27beaSOscar Salvador 142686b27beaSOscar Salvador /* 142786b27beaSOscar Salvador * Check whether node_states[N_NORMAL_MEMORY] will be changed. 142886b27beaSOscar Salvador * If the memory to be offline is within the range 142986b27beaSOscar Salvador * [0..ZONE_NORMAL], and it is the last present memory there, 143086b27beaSOscar Salvador * the zones in that range will become empty after the offlining, 143186b27beaSOscar Salvador * thus we can determine that we need to clear the node from 143286b27beaSOscar Salvador * node_states[N_NORMAL_MEMORY]. 143386b27beaSOscar Salvador */ 143486b27beaSOscar Salvador for (zt = 0; zt <= ZONE_NORMAL; zt++) 143586b27beaSOscar Salvador present_pages += pgdat->node_zones[zt].present_pages; 143686b27beaSOscar Salvador if (zone_idx(zone) <= ZONE_NORMAL && nr_pages >= present_pages) 143786b27beaSOscar Salvador arg->status_change_nid_normal = zone_to_nid(zone); 1438d9713679SLai Jiangshan 14396715ddf9SLai Jiangshan #ifdef CONFIG_HIGHMEM 14406715ddf9SLai Jiangshan /* 144186b27beaSOscar Salvador * node_states[N_HIGH_MEMORY] contains nodes which 144286b27beaSOscar Salvador * have normal memory or high memory. 144386b27beaSOscar Salvador * Here we add the present_pages belonging to ZONE_HIGHMEM. 144486b27beaSOscar Salvador * If the zone is within the range of [0..ZONE_HIGHMEM), and 144586b27beaSOscar Salvador * we determine that the zones in that range become empty, 144686b27beaSOscar Salvador * we need to clear the node for N_HIGH_MEMORY. 14476715ddf9SLai Jiangshan */ 144886b27beaSOscar Salvador present_pages += pgdat->node_zones[ZONE_HIGHMEM].present_pages; 144986b27beaSOscar Salvador if (zone_idx(zone) <= ZONE_HIGHMEM && nr_pages >= present_pages) 14506715ddf9SLai Jiangshan arg->status_change_nid_high = zone_to_nid(zone); 14516715ddf9SLai Jiangshan #endif 14526715ddf9SLai Jiangshan 1453d9713679SLai Jiangshan /* 145486b27beaSOscar Salvador * We have accounted the pages from [0..ZONE_NORMAL), and 145586b27beaSOscar Salvador * in case of CONFIG_HIGHMEM the pages from ZONE_HIGHMEM 145686b27beaSOscar Salvador * as well. 145786b27beaSOscar Salvador * Here we count the possible pages from ZONE_MOVABLE. 145886b27beaSOscar Salvador * If after having accounted all the pages, we see that the nr_pages 145986b27beaSOscar Salvador * to be offlined is over or equal to the accounted pages, 146086b27beaSOscar Salvador * we know that the node will become empty, and so, we can clear 146186b27beaSOscar Salvador * it for N_MEMORY as well. 1462d9713679SLai Jiangshan */ 146386b27beaSOscar Salvador present_pages += pgdat->node_zones[ZONE_MOVABLE].present_pages; 1464d9713679SLai Jiangshan 1465d9713679SLai Jiangshan if (nr_pages >= present_pages) 1466d9713679SLai Jiangshan arg->status_change_nid = zone_to_nid(zone); 1467d9713679SLai Jiangshan } 1468d9713679SLai Jiangshan 1469d9713679SLai Jiangshan static void node_states_clear_node(int node, struct memory_notify *arg) 1470d9713679SLai Jiangshan { 1471d9713679SLai Jiangshan if (arg->status_change_nid_normal >= 0) 1472d9713679SLai Jiangshan node_clear_state(node, N_NORMAL_MEMORY); 1473d9713679SLai Jiangshan 1474cf01f6f5SOscar Salvador if (arg->status_change_nid_high >= 0) 1475d9713679SLai Jiangshan node_clear_state(node, N_HIGH_MEMORY); 14766715ddf9SLai Jiangshan 1477cf01f6f5SOscar Salvador if (arg->status_change_nid >= 0) 14786715ddf9SLai Jiangshan node_clear_state(node, N_MEMORY); 1479d9713679SLai Jiangshan } 1480d9713679SLai Jiangshan 1481a16cee10SWen Congyang static int __ref __offline_pages(unsigned long start_pfn, 1482ecde0f3eSMichal Hocko unsigned long end_pfn) 14830c0e6195SKAMEZAWA Hiroyuki { 1484ecde0f3eSMichal Hocko unsigned long pfn, nr_pages; 14855557c766SMichal Hocko unsigned long offlined_pages = 0; 14869b7ea46aSQian Cai int ret, node, nr_isolate_pageblock; 1487d702909fSCody P Schafer unsigned long flags; 1488a96dfddbSToshi Kani unsigned long valid_start, valid_end; 14890c0e6195SKAMEZAWA Hiroyuki struct zone *zone; 14907b78d335SYasunori Goto struct memory_notify arg; 149179605093SMichal Hocko char *reason; 14920c0e6195SKAMEZAWA Hiroyuki 1493381eab4aSDavid Hildenbrand mem_hotplug_begin(); 1494381eab4aSDavid Hildenbrand 14950c0e6195SKAMEZAWA Hiroyuki /* This makes hotplug much easier...and readable. 14960c0e6195SKAMEZAWA Hiroyuki we assume this for now. .*/ 1497381eab4aSDavid Hildenbrand if (!test_pages_in_a_zone(start_pfn, end_pfn, &valid_start, 1498381eab4aSDavid Hildenbrand &valid_end)) { 149979605093SMichal Hocko ret = -EINVAL; 150079605093SMichal Hocko reason = "multizone range"; 150179605093SMichal Hocko goto failed_removal; 1502381eab4aSDavid Hildenbrand } 15037b78d335SYasunori Goto 1504a96dfddbSToshi Kani zone = page_zone(pfn_to_page(valid_start)); 15057b78d335SYasunori Goto node = zone_to_nid(zone); 15067b78d335SYasunori Goto nr_pages = end_pfn - start_pfn; 15077b78d335SYasunori Goto 15080c0e6195SKAMEZAWA Hiroyuki /* set above range as isolated */ 1509b023f468SWen Congyang ret = start_isolate_page_range(start_pfn, end_pfn, 1510d381c547SMichal Hocko MIGRATE_MOVABLE, 1511d381c547SMichal Hocko SKIP_HWPOISON | REPORT_FAILURE); 15129b7ea46aSQian Cai if (ret < 0) { 151379605093SMichal Hocko reason = "failure to isolate range"; 151479605093SMichal Hocko goto failed_removal; 1515381eab4aSDavid Hildenbrand } 15169b7ea46aSQian Cai nr_isolate_pageblock = ret; 15177b78d335SYasunori Goto 15187b78d335SYasunori Goto arg.start_pfn = start_pfn; 15197b78d335SYasunori Goto arg.nr_pages = nr_pages; 1520d9713679SLai Jiangshan node_states_check_changes_offline(nr_pages, zone, &arg); 15217b78d335SYasunori Goto 15227b78d335SYasunori Goto ret = memory_notify(MEM_GOING_OFFLINE, &arg); 15237b78d335SYasunori Goto ret = notifier_to_errno(ret); 152479605093SMichal Hocko if (ret) { 152579605093SMichal Hocko reason = "notifier failure"; 152679605093SMichal Hocko goto failed_removal_isolated; 152779605093SMichal Hocko } 15287b78d335SYasunori Goto 1529bb8965bdSMichal Hocko do { 1530bb8965bdSMichal Hocko for (pfn = start_pfn; pfn;) { 153179605093SMichal Hocko if (signal_pending(current)) { 1532bb8965bdSMichal Hocko ret = -EINTR; 153379605093SMichal Hocko reason = "signal backoff"; 153479605093SMichal Hocko goto failed_removal_isolated; 153579605093SMichal Hocko } 153672b39cfcSMichal Hocko 15370c0e6195SKAMEZAWA Hiroyuki cond_resched(); 15389852a721SMichal Hocko lru_add_drain_all(); 15390c0e6195SKAMEZAWA Hiroyuki 1540bb8965bdSMichal Hocko pfn = scan_movable_pages(pfn, end_pfn); 1541bb8965bdSMichal Hocko if (pfn) { 1542bb8965bdSMichal Hocko /* 1543bb8965bdSMichal Hocko * TODO: fatal migration failures should bail 1544bb8965bdSMichal Hocko * out 1545bb8965bdSMichal Hocko */ 1546bb8965bdSMichal Hocko do_migrate_range(pfn, end_pfn); 1547bb8965bdSMichal Hocko } 15480c0e6195SKAMEZAWA Hiroyuki } 154972b39cfcSMichal Hocko 1550c8721bbbSNaoya Horiguchi /* 1551bb8965bdSMichal Hocko * Dissolve free hugepages in the memory block before doing 1552bb8965bdSMichal Hocko * offlining actually in order to make hugetlbfs's object 1553bb8965bdSMichal Hocko * counting consistent. 1554c8721bbbSNaoya Horiguchi */ 1555082d5b6bSGerald Schaefer ret = dissolve_free_huge_pages(start_pfn, end_pfn); 155679605093SMichal Hocko if (ret) { 155779605093SMichal Hocko reason = "failure to dissolve huge pages"; 155879605093SMichal Hocko goto failed_removal_isolated; 155979605093SMichal Hocko } 15600c0e6195SKAMEZAWA Hiroyuki /* check again */ 15615557c766SMichal Hocko ret = walk_system_ram_range(start_pfn, end_pfn - start_pfn, 15625557c766SMichal Hocko NULL, check_pages_isolated_cb); 15635557c766SMichal Hocko } while (ret); 1564bb8965bdSMichal Hocko 1565b3834be5SAdam Buchbinder /* Ok, all of our target is isolated. 15660c0e6195SKAMEZAWA Hiroyuki We cannot do rollback at this point. */ 15675557c766SMichal Hocko walk_system_ram_range(start_pfn, end_pfn - start_pfn, 15685557c766SMichal Hocko &offlined_pages, offline_isolated_pages_cb); 15695557c766SMichal Hocko pr_info("Offlined Pages %ld\n", offlined_pages); 15709b7ea46aSQian Cai /* 15719b7ea46aSQian Cai * Onlining will reset pagetype flags and makes migrate type 15729b7ea46aSQian Cai * MOVABLE, so just need to decrease the number of isolated 15739b7ea46aSQian Cai * pageblocks zone counter here. 15749b7ea46aSQian Cai */ 15759b7ea46aSQian Cai spin_lock_irqsave(&zone->lock, flags); 15769b7ea46aSQian Cai zone->nr_isolate_pageblock -= nr_isolate_pageblock; 15779b7ea46aSQian Cai spin_unlock_irqrestore(&zone->lock, flags); 15789b7ea46aSQian Cai 15790c0e6195SKAMEZAWA Hiroyuki /* removal success */ 15803dcc0571SJiang Liu adjust_managed_page_count(pfn_to_page(start_pfn), -offlined_pages); 15810c0e6195SKAMEZAWA Hiroyuki zone->present_pages -= offlined_pages; 1582d702909fSCody P Schafer 1583d702909fSCody P Schafer pgdat_resize_lock(zone->zone_pgdat, &flags); 15840c0e6195SKAMEZAWA Hiroyuki zone->zone_pgdat->node_present_pages -= offlined_pages; 1585d702909fSCody P Schafer pgdat_resize_unlock(zone->zone_pgdat, &flags); 15867b78d335SYasunori Goto 15871b79acc9SKOSAKI Motohiro init_per_zone_wmark_min(); 15881b79acc9SKOSAKI Motohiro 15891e8537baSXishi Qiu if (!populated_zone(zone)) { 1590340175b7SJiang Liu zone_pcp_reset(zone); 159172675e13SMichal Hocko build_all_zonelists(NULL); 15921e8537baSXishi Qiu } else 15931e8537baSXishi Qiu zone_pcp_update(zone); 1594340175b7SJiang Liu 1595d9713679SLai Jiangshan node_states_clear_node(node, &arg); 1596698b1b30SVlastimil Babka if (arg.status_change_nid >= 0) { 15978fe23e05SDavid Rientjes kswapd_stop(node); 1598698b1b30SVlastimil Babka kcompactd_stop(node); 1599698b1b30SVlastimil Babka } 1600bce7394aSMinchan Kim 16010c0e6195SKAMEZAWA Hiroyuki vm_total_pages = nr_free_pagecache_pages(); 16020c0e6195SKAMEZAWA Hiroyuki writeback_set_ratelimit(); 16037b78d335SYasunori Goto 16047b78d335SYasunori Goto memory_notify(MEM_OFFLINE, &arg); 1605381eab4aSDavid Hildenbrand mem_hotplug_done(); 16060c0e6195SKAMEZAWA Hiroyuki return 0; 16070c0e6195SKAMEZAWA Hiroyuki 160879605093SMichal Hocko failed_removal_isolated: 160979605093SMichal Hocko undo_isolate_page_range(start_pfn, end_pfn, MIGRATE_MOVABLE); 1610c4efe484SQian Cai memory_notify(MEM_CANCEL_OFFLINE, &arg); 16110c0e6195SKAMEZAWA Hiroyuki failed_removal: 161279605093SMichal Hocko pr_debug("memory offlining [mem %#010llx-%#010llx] failed due to %s\n", 1613a62e2f4fSBjorn Helgaas (unsigned long long) start_pfn << PAGE_SHIFT, 161479605093SMichal Hocko ((unsigned long long) end_pfn << PAGE_SHIFT) - 1, 161579605093SMichal Hocko reason); 16160c0e6195SKAMEZAWA Hiroyuki /* pushback to free area */ 1617381eab4aSDavid Hildenbrand mem_hotplug_done(); 16180c0e6195SKAMEZAWA Hiroyuki return ret; 16190c0e6195SKAMEZAWA Hiroyuki } 162071088785SBadari Pulavarty 1621a16cee10SWen Congyang int offline_pages(unsigned long start_pfn, unsigned long nr_pages) 1622a16cee10SWen Congyang { 1623ecde0f3eSMichal Hocko return __offline_pages(start_pfn, start_pfn + nr_pages); 1624a16cee10SWen Congyang } 1625a16cee10SWen Congyang 1626d6de9d53SXishi Qiu static int check_memblock_offlined_cb(struct memory_block *mem, void *arg) 1627bbc76be6SWen Congyang { 1628bbc76be6SWen Congyang int ret = !is_memblock_offlined(mem); 1629bbc76be6SWen Congyang 1630349daa0fSRandy Dunlap if (unlikely(ret)) { 1631349daa0fSRandy Dunlap phys_addr_t beginpa, endpa; 1632349daa0fSRandy Dunlap 1633349daa0fSRandy Dunlap beginpa = PFN_PHYS(section_nr_to_pfn(mem->start_section_nr)); 1634b6c88d3bSDavid Hildenbrand endpa = beginpa + memory_block_size_bytes() - 1; 1635756a025fSJoe Perches pr_warn("removing memory fails, because memory [%pa-%pa] is onlined\n", 1636349daa0fSRandy Dunlap &beginpa, &endpa); 1637bbc76be6SWen Congyang 1638eca499abSPavel Tatashin return -EBUSY; 1639eca499abSPavel Tatashin } 1640eca499abSPavel Tatashin return 0; 1641bbc76be6SWen Congyang } 1642bbc76be6SWen Congyang 16430f1cfe9dSToshi Kani static int check_cpu_on_node(pg_data_t *pgdat) 164460a5a19eSTang Chen { 164560a5a19eSTang Chen int cpu; 164660a5a19eSTang Chen 164760a5a19eSTang Chen for_each_present_cpu(cpu) { 164860a5a19eSTang Chen if (cpu_to_node(cpu) == pgdat->node_id) 164960a5a19eSTang Chen /* 165060a5a19eSTang Chen * the cpu on this node isn't removed, and we can't 165160a5a19eSTang Chen * offline this node. 165260a5a19eSTang Chen */ 165360a5a19eSTang Chen return -EBUSY; 165460a5a19eSTang Chen } 165560a5a19eSTang Chen 165660a5a19eSTang Chen return 0; 165760a5a19eSTang Chen } 165860a5a19eSTang Chen 16592c91f8fcSDavid Hildenbrand static int check_no_memblock_for_node_cb(struct memory_block *mem, void *arg) 16602c91f8fcSDavid Hildenbrand { 16612c91f8fcSDavid Hildenbrand int nid = *(int *)arg; 16622c91f8fcSDavid Hildenbrand 16632c91f8fcSDavid Hildenbrand /* 16642c91f8fcSDavid Hildenbrand * If a memory block belongs to multiple nodes, the stored nid is not 16652c91f8fcSDavid Hildenbrand * reliable. However, such blocks are always online (e.g., cannot get 16662c91f8fcSDavid Hildenbrand * offlined) and, therefore, are still spanned by the node. 16672c91f8fcSDavid Hildenbrand */ 16682c91f8fcSDavid Hildenbrand return mem->nid == nid ? -EEXIST : 0; 16692c91f8fcSDavid Hildenbrand } 16702c91f8fcSDavid Hildenbrand 16710f1cfe9dSToshi Kani /** 16720f1cfe9dSToshi Kani * try_offline_node 1673e8b098fcSMike Rapoport * @nid: the node ID 16740f1cfe9dSToshi Kani * 16750f1cfe9dSToshi Kani * Offline a node if all memory sections and cpus of the node are removed. 16760f1cfe9dSToshi Kani * 16770f1cfe9dSToshi Kani * NOTE: The caller must call lock_device_hotplug() to serialize hotplug 16780f1cfe9dSToshi Kani * and online/offline operations before this call. 16790f1cfe9dSToshi Kani */ 168090b30cdcSWen Congyang void try_offline_node(int nid) 168160a5a19eSTang Chen { 1682d822b86aSWen Congyang pg_data_t *pgdat = NODE_DATA(nid); 16832c91f8fcSDavid Hildenbrand int rc; 168460a5a19eSTang Chen 168560a5a19eSTang Chen /* 16862c91f8fcSDavid Hildenbrand * If the node still spans pages (especially ZONE_DEVICE), don't 16872c91f8fcSDavid Hildenbrand * offline it. A node spans memory after move_pfn_range_to_zone(), 16882c91f8fcSDavid Hildenbrand * e.g., after the memory block was onlined. 168960a5a19eSTang Chen */ 16902c91f8fcSDavid Hildenbrand if (pgdat->node_spanned_pages) 169160a5a19eSTang Chen return; 16922c91f8fcSDavid Hildenbrand 16932c91f8fcSDavid Hildenbrand /* 16942c91f8fcSDavid Hildenbrand * Especially offline memory blocks might not be spanned by the 16952c91f8fcSDavid Hildenbrand * node. They will get spanned by the node once they get onlined. 16962c91f8fcSDavid Hildenbrand * However, they link to the node in sysfs and can get onlined later. 16972c91f8fcSDavid Hildenbrand */ 16982c91f8fcSDavid Hildenbrand rc = for_each_memory_block(&nid, check_no_memblock_for_node_cb); 16992c91f8fcSDavid Hildenbrand if (rc) 17002c91f8fcSDavid Hildenbrand return; 170160a5a19eSTang Chen 170246a3679bSMichal Hocko if (check_cpu_on_node(pgdat)) 170360a5a19eSTang Chen return; 170460a5a19eSTang Chen 170560a5a19eSTang Chen /* 170660a5a19eSTang Chen * all memory/cpu of this node are removed, we can offline this 170760a5a19eSTang Chen * node now. 170860a5a19eSTang Chen */ 170960a5a19eSTang Chen node_set_offline(nid); 171060a5a19eSTang Chen unregister_one_node(nid); 171160a5a19eSTang Chen } 171290b30cdcSWen Congyang EXPORT_SYMBOL(try_offline_node); 171360a5a19eSTang Chen 1714d9eb1417SDavid Hildenbrand static void __release_memory_resource(resource_size_t start, 1715d9eb1417SDavid Hildenbrand resource_size_t size) 1716d9eb1417SDavid Hildenbrand { 1717d9eb1417SDavid Hildenbrand int ret; 1718d9eb1417SDavid Hildenbrand 1719d9eb1417SDavid Hildenbrand /* 1720d9eb1417SDavid Hildenbrand * When removing memory in the same granularity as it was added, 1721d9eb1417SDavid Hildenbrand * this function never fails. It might only fail if resources 1722d9eb1417SDavid Hildenbrand * have to be adjusted or split. We'll ignore the error, as 1723d9eb1417SDavid Hildenbrand * removing of memory cannot fail. 1724d9eb1417SDavid Hildenbrand */ 1725d9eb1417SDavid Hildenbrand ret = release_mem_region_adjustable(&iomem_resource, start, size); 1726d9eb1417SDavid Hildenbrand if (ret) { 1727d9eb1417SDavid Hildenbrand resource_size_t endres = start + size - 1; 1728d9eb1417SDavid Hildenbrand 1729d9eb1417SDavid Hildenbrand pr_warn("Unable to release resource <%pa-%pa> (%d)\n", 1730d9eb1417SDavid Hildenbrand &start, &endres, ret); 1731d9eb1417SDavid Hildenbrand } 1732d9eb1417SDavid Hildenbrand } 1733d9eb1417SDavid Hildenbrand 1734eca499abSPavel Tatashin static int __ref try_remove_memory(int nid, u64 start, u64 size) 1735bbc76be6SWen Congyang { 1736eca499abSPavel Tatashin int rc = 0; 1737993c1aadSWen Congyang 173827356f54SToshi Kani BUG_ON(check_hotplug_memory_range(start, size)); 173927356f54SToshi Kani 1740bfc8c901SVladimir Davydov mem_hotplug_begin(); 17416677e3eaSYasuaki Ishimatsu 17426677e3eaSYasuaki Ishimatsu /* 1743242831ebSRafael J. Wysocki * All memory blocks must be offlined before removing memory. Check 1744eca499abSPavel Tatashin * whether all memory blocks in question are offline and return error 1745242831ebSRafael J. Wysocki * if this is not the case. 17466677e3eaSYasuaki Ishimatsu */ 1747fbcf73ceSDavid Hildenbrand rc = walk_memory_blocks(start, size, NULL, check_memblock_offlined_cb); 1748eca499abSPavel Tatashin if (rc) 1749eca499abSPavel Tatashin goto done; 17506677e3eaSYasuaki Ishimatsu 175146c66c4bSYasuaki Ishimatsu /* remove memmap entry */ 175246c66c4bSYasuaki Ishimatsu firmware_map_remove(start, start + size, "System RAM"); 175346c66c4bSYasuaki Ishimatsu 17544c4b7f9bSDavid Hildenbrand /* remove memory block devices before removing memory */ 17554c4b7f9bSDavid Hildenbrand remove_memory_block_devices(start, size); 17564c4b7f9bSDavid Hildenbrand 17572c2a5af6SOscar Salvador arch_remove_memory(nid, start, size, NULL); 1758*32d1fe8fSAnshuman Khandual memblock_free(start, size); 1759*32d1fe8fSAnshuman Khandual memblock_remove(start, size); 1760d9eb1417SDavid Hildenbrand __release_memory_resource(start, size); 176124d335caSWen Congyang 176260a5a19eSTang Chen try_offline_node(nid); 176360a5a19eSTang Chen 1764eca499abSPavel Tatashin done: 1765bfc8c901SVladimir Davydov mem_hotplug_done(); 1766eca499abSPavel Tatashin return rc; 176771088785SBadari Pulavarty } 1768d15e5926SDavid Hildenbrand 1769eca499abSPavel Tatashin /** 1770eca499abSPavel Tatashin * remove_memory 1771eca499abSPavel Tatashin * @nid: the node ID 1772eca499abSPavel Tatashin * @start: physical address of the region to remove 1773eca499abSPavel Tatashin * @size: size of the region to remove 1774eca499abSPavel Tatashin * 1775eca499abSPavel Tatashin * NOTE: The caller must call lock_device_hotplug() to serialize hotplug 1776eca499abSPavel Tatashin * and online/offline operations before this call, as required by 1777eca499abSPavel Tatashin * try_offline_node(). 1778eca499abSPavel Tatashin */ 1779eca499abSPavel Tatashin void __remove_memory(int nid, u64 start, u64 size) 1780d15e5926SDavid Hildenbrand { 1781eca499abSPavel Tatashin 1782eca499abSPavel Tatashin /* 178329a90db9SSouptick Joarder * trigger BUG() if some memory is not offlined prior to calling this 1784eca499abSPavel Tatashin * function 1785eca499abSPavel Tatashin */ 1786eca499abSPavel Tatashin if (try_remove_memory(nid, start, size)) 1787eca499abSPavel Tatashin BUG(); 1788eca499abSPavel Tatashin } 1789eca499abSPavel Tatashin 1790eca499abSPavel Tatashin /* 1791eca499abSPavel Tatashin * Remove memory if every memory block is offline, otherwise return -EBUSY is 1792eca499abSPavel Tatashin * some memory is not offline 1793eca499abSPavel Tatashin */ 1794eca499abSPavel Tatashin int remove_memory(int nid, u64 start, u64 size) 1795eca499abSPavel Tatashin { 1796eca499abSPavel Tatashin int rc; 1797eca499abSPavel Tatashin 1798d15e5926SDavid Hildenbrand lock_device_hotplug(); 1799eca499abSPavel Tatashin rc = try_remove_memory(nid, start, size); 1800d15e5926SDavid Hildenbrand unlock_device_hotplug(); 1801eca499abSPavel Tatashin 1802eca499abSPavel Tatashin return rc; 1803d15e5926SDavid Hildenbrand } 180471088785SBadari Pulavarty EXPORT_SYMBOL_GPL(remove_memory); 1805aba6efc4SRafael J. Wysocki #endif /* CONFIG_MEMORY_HOTREMOVE */ 1806