13947be19SDave Hansen /* 23947be19SDave Hansen * linux/mm/memory_hotplug.c 33947be19SDave Hansen * 43947be19SDave Hansen * Copyright (C) 53947be19SDave Hansen */ 63947be19SDave Hansen 73947be19SDave Hansen #include <linux/stddef.h> 83947be19SDave Hansen #include <linux/mm.h> 93947be19SDave Hansen #include <linux/swap.h> 103947be19SDave Hansen #include <linux/interrupt.h> 113947be19SDave Hansen #include <linux/pagemap.h> 123947be19SDave Hansen #include <linux/compiler.h> 13b95f1b31SPaul Gortmaker #include <linux/export.h> 143947be19SDave Hansen #include <linux/pagevec.h> 152d1d43f6SChandra Seetharaman #include <linux/writeback.h> 163947be19SDave Hansen #include <linux/slab.h> 173947be19SDave Hansen #include <linux/sysctl.h> 183947be19SDave Hansen #include <linux/cpu.h> 193947be19SDave Hansen #include <linux/memory.h> 203947be19SDave Hansen #include <linux/memory_hotplug.h> 213947be19SDave Hansen #include <linux/highmem.h> 223947be19SDave Hansen #include <linux/vmalloc.h> 230a547039SKAMEZAWA Hiroyuki #include <linux/ioport.h> 240c0e6195SKAMEZAWA Hiroyuki #include <linux/delay.h> 250c0e6195SKAMEZAWA Hiroyuki #include <linux/migrate.h> 260c0e6195SKAMEZAWA Hiroyuki #include <linux/page-isolation.h> 2771088785SBadari Pulavarty #include <linux/pfn.h> 286ad696d2SAndi Kleen #include <linux/suspend.h> 296d9c285aSKOSAKI Motohiro #include <linux/mm_inline.h> 30d96ae530Sakpm@linux-foundation.org #include <linux/firmware-map.h> 3160a5a19eSTang Chen #include <linux/stop_machine.h> 32c8721bbbSNaoya Horiguchi #include <linux/hugetlb.h> 33c5320926STang Chen #include <linux/memblock.h> 34f784a3f1STang Chen #include <linux/bootmem.h> 353947be19SDave Hansen 363947be19SDave Hansen #include <asm/tlbflush.h> 373947be19SDave Hansen 381e5ad9a3SAdrian Bunk #include "internal.h" 391e5ad9a3SAdrian Bunk 409d0ad8caSDaniel Kiper /* 419d0ad8caSDaniel Kiper * online_page_callback contains pointer to current page onlining function. 429d0ad8caSDaniel Kiper * Initially it is generic_online_page(). If it is required it could be 439d0ad8caSDaniel Kiper * changed by calling set_online_page_callback() for callback registration 449d0ad8caSDaniel Kiper * and restore_online_page_callback() for generic callback restore. 459d0ad8caSDaniel Kiper */ 469d0ad8caSDaniel Kiper 479d0ad8caSDaniel Kiper static void generic_online_page(struct page *page); 489d0ad8caSDaniel Kiper 499d0ad8caSDaniel Kiper static online_page_callback_t online_page_callback = generic_online_page; 50bfc8c901SVladimir Davydov static DEFINE_MUTEX(online_page_callback_lock); 519d0ad8caSDaniel Kiper 52bfc8c901SVladimir Davydov /* The same as the cpu_hotplug lock, but for memory hotplug. */ 53bfc8c901SVladimir Davydov static struct { 54bfc8c901SVladimir Davydov struct task_struct *active_writer; 55bfc8c901SVladimir Davydov struct mutex lock; /* Synchronizes accesses to refcount, */ 56bfc8c901SVladimir Davydov /* 57bfc8c901SVladimir Davydov * Also blocks the new readers during 58bfc8c901SVladimir Davydov * an ongoing mem hotplug operation. 59bfc8c901SVladimir Davydov */ 60bfc8c901SVladimir Davydov int refcount; 6120d6c96bSKOSAKI Motohiro 62bfc8c901SVladimir Davydov #ifdef CONFIG_DEBUG_LOCK_ALLOC 63bfc8c901SVladimir Davydov struct lockdep_map dep_map; 64bfc8c901SVladimir Davydov #endif 65bfc8c901SVladimir Davydov } mem_hotplug = { 66bfc8c901SVladimir Davydov .active_writer = NULL, 67bfc8c901SVladimir Davydov .lock = __MUTEX_INITIALIZER(mem_hotplug.lock), 68bfc8c901SVladimir Davydov .refcount = 0, 69bfc8c901SVladimir Davydov #ifdef CONFIG_DEBUG_LOCK_ALLOC 70bfc8c901SVladimir Davydov .dep_map = {.name = "mem_hotplug.lock" }, 71bfc8c901SVladimir Davydov #endif 72bfc8c901SVladimir Davydov }; 73bfc8c901SVladimir Davydov 74bfc8c901SVladimir Davydov /* Lockdep annotations for get/put_online_mems() and mem_hotplug_begin/end() */ 75bfc8c901SVladimir Davydov #define memhp_lock_acquire_read() lock_map_acquire_read(&mem_hotplug.dep_map) 76bfc8c901SVladimir Davydov #define memhp_lock_acquire() lock_map_acquire(&mem_hotplug.dep_map) 77bfc8c901SVladimir Davydov #define memhp_lock_release() lock_map_release(&mem_hotplug.dep_map) 78bfc8c901SVladimir Davydov 79bfc8c901SVladimir Davydov void get_online_mems(void) 8020d6c96bSKOSAKI Motohiro { 81bfc8c901SVladimir Davydov might_sleep(); 82bfc8c901SVladimir Davydov if (mem_hotplug.active_writer == current) 83bfc8c901SVladimir Davydov return; 84bfc8c901SVladimir Davydov memhp_lock_acquire_read(); 85bfc8c901SVladimir Davydov mutex_lock(&mem_hotplug.lock); 86bfc8c901SVladimir Davydov mem_hotplug.refcount++; 87bfc8c901SVladimir Davydov mutex_unlock(&mem_hotplug.lock); 88bfc8c901SVladimir Davydov 8920d6c96bSKOSAKI Motohiro } 9020d6c96bSKOSAKI Motohiro 91bfc8c901SVladimir Davydov void put_online_mems(void) 9220d6c96bSKOSAKI Motohiro { 93bfc8c901SVladimir Davydov if (mem_hotplug.active_writer == current) 94bfc8c901SVladimir Davydov return; 95bfc8c901SVladimir Davydov mutex_lock(&mem_hotplug.lock); 96bfc8c901SVladimir Davydov 97bfc8c901SVladimir Davydov if (WARN_ON(!mem_hotplug.refcount)) 98bfc8c901SVladimir Davydov mem_hotplug.refcount++; /* try to fix things up */ 99bfc8c901SVladimir Davydov 100bfc8c901SVladimir Davydov if (!--mem_hotplug.refcount && unlikely(mem_hotplug.active_writer)) 101bfc8c901SVladimir Davydov wake_up_process(mem_hotplug.active_writer); 102bfc8c901SVladimir Davydov mutex_unlock(&mem_hotplug.lock); 103bfc8c901SVladimir Davydov memhp_lock_release(); 104bfc8c901SVladimir Davydov 10520d6c96bSKOSAKI Motohiro } 10620d6c96bSKOSAKI Motohiro 10730467e0bSDavid Rientjes void mem_hotplug_begin(void) 108bfc8c901SVladimir Davydov { 109bfc8c901SVladimir Davydov mem_hotplug.active_writer = current; 110bfc8c901SVladimir Davydov 111bfc8c901SVladimir Davydov memhp_lock_acquire(); 112bfc8c901SVladimir Davydov for (;;) { 113bfc8c901SVladimir Davydov mutex_lock(&mem_hotplug.lock); 114bfc8c901SVladimir Davydov if (likely(!mem_hotplug.refcount)) 115bfc8c901SVladimir Davydov break; 116bfc8c901SVladimir Davydov __set_current_state(TASK_UNINTERRUPTIBLE); 117bfc8c901SVladimir Davydov mutex_unlock(&mem_hotplug.lock); 118bfc8c901SVladimir Davydov schedule(); 119bfc8c901SVladimir Davydov } 120bfc8c901SVladimir Davydov } 121bfc8c901SVladimir Davydov 12230467e0bSDavid Rientjes void mem_hotplug_done(void) 123bfc8c901SVladimir Davydov { 124bfc8c901SVladimir Davydov mem_hotplug.active_writer = NULL; 125bfc8c901SVladimir Davydov mutex_unlock(&mem_hotplug.lock); 126bfc8c901SVladimir Davydov memhp_lock_release(); 127bfc8c901SVladimir Davydov } 12820d6c96bSKOSAKI Motohiro 12945e0b78bSKeith Mannthey /* add this memory to iomem resource */ 13045e0b78bSKeith Mannthey static struct resource *register_memory_resource(u64 start, u64 size) 13145e0b78bSKeith Mannthey { 13245e0b78bSKeith Mannthey struct resource *res; 13345e0b78bSKeith Mannthey res = kzalloc(sizeof(struct resource), GFP_KERNEL); 134*6f754ba4SVitaly Kuznetsov if (!res) 135*6f754ba4SVitaly Kuznetsov return ERR_PTR(-ENOMEM); 13645e0b78bSKeith Mannthey 13745e0b78bSKeith Mannthey res->name = "System RAM"; 13845e0b78bSKeith Mannthey res->start = start; 13945e0b78bSKeith Mannthey res->end = start + size - 1; 140887c3cb1SYasunori Goto res->flags = IORESOURCE_MEM | IORESOURCE_BUSY; 14145e0b78bSKeith Mannthey if (request_resource(&iomem_resource, res) < 0) { 1424996eed8SToshi Kani pr_debug("System RAM resource %pR cannot be added\n", res); 14345e0b78bSKeith Mannthey kfree(res); 144*6f754ba4SVitaly Kuznetsov return ERR_PTR(-EEXIST); 14545e0b78bSKeith Mannthey } 14645e0b78bSKeith Mannthey return res; 14745e0b78bSKeith Mannthey } 14845e0b78bSKeith Mannthey 14945e0b78bSKeith Mannthey static void release_memory_resource(struct resource *res) 15045e0b78bSKeith Mannthey { 15145e0b78bSKeith Mannthey if (!res) 15245e0b78bSKeith Mannthey return; 15345e0b78bSKeith Mannthey release_resource(res); 15445e0b78bSKeith Mannthey kfree(res); 15545e0b78bSKeith Mannthey return; 15645e0b78bSKeith Mannthey } 15745e0b78bSKeith Mannthey 15853947027SKeith Mannthey #ifdef CONFIG_MEMORY_HOTPLUG_SPARSE 15946723bfaSYasuaki Ishimatsu void get_page_bootmem(unsigned long info, struct page *page, 1605f24ce5fSAndrea Arcangeli unsigned long type) 16104753278SYasunori Goto { 1625f24ce5fSAndrea Arcangeli page->lru.next = (struct list_head *) type; 16304753278SYasunori Goto SetPagePrivate(page); 16404753278SYasunori Goto set_page_private(page, info); 16504753278SYasunori Goto atomic_inc(&page->_count); 16604753278SYasunori Goto } 16704753278SYasunori Goto 168170a5a7eSJiang Liu void put_page_bootmem(struct page *page) 16904753278SYasunori Goto { 1705f24ce5fSAndrea Arcangeli unsigned long type; 17104753278SYasunori Goto 1725f24ce5fSAndrea Arcangeli type = (unsigned long) page->lru.next; 1735f24ce5fSAndrea Arcangeli BUG_ON(type < MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE || 1745f24ce5fSAndrea Arcangeli type > MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE); 17504753278SYasunori Goto 17604753278SYasunori Goto if (atomic_dec_return(&page->_count) == 1) { 17704753278SYasunori Goto ClearPagePrivate(page); 17804753278SYasunori Goto set_page_private(page, 0); 1795f24ce5fSAndrea Arcangeli INIT_LIST_HEAD(&page->lru); 180170a5a7eSJiang Liu free_reserved_page(page); 18104753278SYasunori Goto } 18204753278SYasunori Goto } 18304753278SYasunori Goto 18446723bfaSYasuaki Ishimatsu #ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE 18546723bfaSYasuaki Ishimatsu #ifndef CONFIG_SPARSEMEM_VMEMMAP 186d92bc318SAdrian Bunk static void register_page_bootmem_info_section(unsigned long start_pfn) 18704753278SYasunori Goto { 18804753278SYasunori Goto unsigned long *usemap, mapsize, section_nr, i; 18904753278SYasunori Goto struct mem_section *ms; 19004753278SYasunori Goto struct page *page, *memmap; 19104753278SYasunori Goto 19204753278SYasunori Goto section_nr = pfn_to_section_nr(start_pfn); 19304753278SYasunori Goto ms = __nr_to_section(section_nr); 19404753278SYasunori Goto 19504753278SYasunori Goto /* Get section's memmap address */ 19604753278SYasunori Goto memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr); 19704753278SYasunori Goto 19804753278SYasunori Goto /* 19904753278SYasunori Goto * Get page for the memmap's phys address 20004753278SYasunori Goto * XXX: need more consideration for sparse_vmemmap... 20104753278SYasunori Goto */ 20204753278SYasunori Goto page = virt_to_page(memmap); 20304753278SYasunori Goto mapsize = sizeof(struct page) * PAGES_PER_SECTION; 20404753278SYasunori Goto mapsize = PAGE_ALIGN(mapsize) >> PAGE_SHIFT; 20504753278SYasunori Goto 20604753278SYasunori Goto /* remember memmap's page */ 20704753278SYasunori Goto for (i = 0; i < mapsize; i++, page++) 20804753278SYasunori Goto get_page_bootmem(section_nr, page, SECTION_INFO); 20904753278SYasunori Goto 21004753278SYasunori Goto usemap = __nr_to_section(section_nr)->pageblock_flags; 21104753278SYasunori Goto page = virt_to_page(usemap); 21204753278SYasunori Goto 21304753278SYasunori Goto mapsize = PAGE_ALIGN(usemap_size()) >> PAGE_SHIFT; 21404753278SYasunori Goto 21504753278SYasunori Goto for (i = 0; i < mapsize; i++, page++) 216af370fb8SYasunori Goto get_page_bootmem(section_nr, page, MIX_SECTION_INFO); 21704753278SYasunori Goto 21804753278SYasunori Goto } 21946723bfaSYasuaki Ishimatsu #else /* CONFIG_SPARSEMEM_VMEMMAP */ 22046723bfaSYasuaki Ishimatsu static void register_page_bootmem_info_section(unsigned long start_pfn) 22146723bfaSYasuaki Ishimatsu { 22246723bfaSYasuaki Ishimatsu unsigned long *usemap, mapsize, section_nr, i; 22346723bfaSYasuaki Ishimatsu struct mem_section *ms; 22446723bfaSYasuaki Ishimatsu struct page *page, *memmap; 22546723bfaSYasuaki Ishimatsu 22646723bfaSYasuaki Ishimatsu if (!pfn_valid(start_pfn)) 22746723bfaSYasuaki Ishimatsu return; 22846723bfaSYasuaki Ishimatsu 22946723bfaSYasuaki Ishimatsu section_nr = pfn_to_section_nr(start_pfn); 23046723bfaSYasuaki Ishimatsu ms = __nr_to_section(section_nr); 23146723bfaSYasuaki Ishimatsu 23246723bfaSYasuaki Ishimatsu memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr); 23346723bfaSYasuaki Ishimatsu 23446723bfaSYasuaki Ishimatsu register_page_bootmem_memmap(section_nr, memmap, PAGES_PER_SECTION); 23546723bfaSYasuaki Ishimatsu 23646723bfaSYasuaki Ishimatsu usemap = __nr_to_section(section_nr)->pageblock_flags; 23746723bfaSYasuaki Ishimatsu page = virt_to_page(usemap); 23846723bfaSYasuaki Ishimatsu 23946723bfaSYasuaki Ishimatsu mapsize = PAGE_ALIGN(usemap_size()) >> PAGE_SHIFT; 24046723bfaSYasuaki Ishimatsu 24146723bfaSYasuaki Ishimatsu for (i = 0; i < mapsize; i++, page++) 24246723bfaSYasuaki Ishimatsu get_page_bootmem(section_nr, page, MIX_SECTION_INFO); 24346723bfaSYasuaki Ishimatsu } 24446723bfaSYasuaki Ishimatsu #endif /* !CONFIG_SPARSEMEM_VMEMMAP */ 24504753278SYasunori Goto 24604753278SYasunori Goto void register_page_bootmem_info_node(struct pglist_data *pgdat) 24704753278SYasunori Goto { 24804753278SYasunori Goto unsigned long i, pfn, end_pfn, nr_pages; 24904753278SYasunori Goto int node = pgdat->node_id; 25004753278SYasunori Goto struct page *page; 25104753278SYasunori Goto struct zone *zone; 25204753278SYasunori Goto 25304753278SYasunori Goto nr_pages = PAGE_ALIGN(sizeof(struct pglist_data)) >> PAGE_SHIFT; 25404753278SYasunori Goto page = virt_to_page(pgdat); 25504753278SYasunori Goto 25604753278SYasunori Goto for (i = 0; i < nr_pages; i++, page++) 25704753278SYasunori Goto get_page_bootmem(node, page, NODE_INFO); 25804753278SYasunori Goto 25904753278SYasunori Goto zone = &pgdat->node_zones[0]; 26004753278SYasunori Goto for (; zone < pgdat->node_zones + MAX_NR_ZONES - 1; zone++) { 261139c2d75SXishi Qiu if (zone_is_initialized(zone)) { 26204753278SYasunori Goto nr_pages = zone->wait_table_hash_nr_entries 26304753278SYasunori Goto * sizeof(wait_queue_head_t); 26404753278SYasunori Goto nr_pages = PAGE_ALIGN(nr_pages) >> PAGE_SHIFT; 26504753278SYasunori Goto page = virt_to_page(zone->wait_table); 26604753278SYasunori Goto 26704753278SYasunori Goto for (i = 0; i < nr_pages; i++, page++) 26804753278SYasunori Goto get_page_bootmem(node, page, NODE_INFO); 26904753278SYasunori Goto } 27004753278SYasunori Goto } 27104753278SYasunori Goto 27204753278SYasunori Goto pfn = pgdat->node_start_pfn; 273c1f19495SCody P Schafer end_pfn = pgdat_end_pfn(pgdat); 27404753278SYasunori Goto 2757e9f5eb0STang Chen /* register section info */ 276f14851afSqiuxishi for (; pfn < end_pfn; pfn += PAGES_PER_SECTION) { 277f14851afSqiuxishi /* 278f14851afSqiuxishi * Some platforms can assign the same pfn to multiple nodes - on 279f14851afSqiuxishi * node0 as well as nodeN. To avoid registering a pfn against 280f14851afSqiuxishi * multiple nodes we check that this pfn does not already 2817e9f5eb0STang Chen * reside in some other nodes. 282f14851afSqiuxishi */ 283f14851afSqiuxishi if (pfn_valid(pfn) && (pfn_to_nid(pfn) == node)) 28404753278SYasunori Goto register_page_bootmem_info_section(pfn); 285f14851afSqiuxishi } 28604753278SYasunori Goto } 28746723bfaSYasuaki Ishimatsu #endif /* CONFIG_HAVE_BOOTMEM_INFO_NODE */ 28804753278SYasunori Goto 289f2765404SFabian Frederick static void __meminit grow_zone_span(struct zone *zone, unsigned long start_pfn, 29076cdd58eSHeiko Carstens unsigned long end_pfn) 29176cdd58eSHeiko Carstens { 29276cdd58eSHeiko Carstens unsigned long old_zone_end_pfn; 29376cdd58eSHeiko Carstens 29476cdd58eSHeiko Carstens zone_span_writelock(zone); 29576cdd58eSHeiko Carstens 296c33bc315SXishi Qiu old_zone_end_pfn = zone_end_pfn(zone); 2978080fc03SXishi Qiu if (zone_is_empty(zone) || start_pfn < zone->zone_start_pfn) 29876cdd58eSHeiko Carstens zone->zone_start_pfn = start_pfn; 29976cdd58eSHeiko Carstens 30076cdd58eSHeiko Carstens zone->spanned_pages = max(old_zone_end_pfn, end_pfn) - 30176cdd58eSHeiko Carstens zone->zone_start_pfn; 30276cdd58eSHeiko Carstens 30376cdd58eSHeiko Carstens zone_span_writeunlock(zone); 30476cdd58eSHeiko Carstens } 30576cdd58eSHeiko Carstens 306511c2abaSLai Jiangshan static void resize_zone(struct zone *zone, unsigned long start_pfn, 307511c2abaSLai Jiangshan unsigned long end_pfn) 308511c2abaSLai Jiangshan { 309511c2abaSLai Jiangshan zone_span_writelock(zone); 310511c2abaSLai Jiangshan 311e455a9b9SLai Jiangshan if (end_pfn - start_pfn) { 312511c2abaSLai Jiangshan zone->zone_start_pfn = start_pfn; 313511c2abaSLai Jiangshan zone->spanned_pages = end_pfn - start_pfn; 314e455a9b9SLai Jiangshan } else { 315e455a9b9SLai Jiangshan /* 316e455a9b9SLai Jiangshan * make it consist as free_area_init_core(), 317e455a9b9SLai Jiangshan * if spanned_pages = 0, then keep start_pfn = 0 318e455a9b9SLai Jiangshan */ 319e455a9b9SLai Jiangshan zone->zone_start_pfn = 0; 320e455a9b9SLai Jiangshan zone->spanned_pages = 0; 321e455a9b9SLai Jiangshan } 322511c2abaSLai Jiangshan 323511c2abaSLai Jiangshan zone_span_writeunlock(zone); 324511c2abaSLai Jiangshan } 325511c2abaSLai Jiangshan 326511c2abaSLai Jiangshan static void fix_zone_id(struct zone *zone, unsigned long start_pfn, 327511c2abaSLai Jiangshan unsigned long end_pfn) 328511c2abaSLai Jiangshan { 329511c2abaSLai Jiangshan enum zone_type zid = zone_idx(zone); 330511c2abaSLai Jiangshan int nid = zone->zone_pgdat->node_id; 331511c2abaSLai Jiangshan unsigned long pfn; 332511c2abaSLai Jiangshan 333511c2abaSLai Jiangshan for (pfn = start_pfn; pfn < end_pfn; pfn++) 334511c2abaSLai Jiangshan set_page_links(pfn_to_page(pfn), zid, nid, pfn); 335511c2abaSLai Jiangshan } 336511c2abaSLai Jiangshan 337f6bbb78eSCody P Schafer /* Can fail with -ENOMEM from allocating a wait table with vmalloc() or 3389e43aa2bSSantosh Shilimkar * alloc_bootmem_node_nopanic()/memblock_virt_alloc_node_nopanic() */ 339f6bbb78eSCody P Schafer static int __ref ensure_zone_is_initialized(struct zone *zone, 340f6bbb78eSCody P Schafer unsigned long start_pfn, unsigned long num_pages) 341f6bbb78eSCody P Schafer { 342f6bbb78eSCody P Schafer if (!zone_is_initialized(zone)) 343b171e409SYaowei Bai return init_currently_empty_zone(zone, start_pfn, num_pages); 344b171e409SYaowei Bai 345f6bbb78eSCody P Schafer return 0; 346f6bbb78eSCody P Schafer } 347f6bbb78eSCody P Schafer 348e455a9b9SLai Jiangshan static int __meminit move_pfn_range_left(struct zone *z1, struct zone *z2, 349511c2abaSLai Jiangshan unsigned long start_pfn, unsigned long end_pfn) 350511c2abaSLai Jiangshan { 351e455a9b9SLai Jiangshan int ret; 352511c2abaSLai Jiangshan unsigned long flags; 353e455a9b9SLai Jiangshan unsigned long z1_start_pfn; 354e455a9b9SLai Jiangshan 35564dd1b29SCody P Schafer ret = ensure_zone_is_initialized(z1, start_pfn, end_pfn - start_pfn); 356e455a9b9SLai Jiangshan if (ret) 357e455a9b9SLai Jiangshan return ret; 358511c2abaSLai Jiangshan 359511c2abaSLai Jiangshan pgdat_resize_lock(z1->zone_pgdat, &flags); 360511c2abaSLai Jiangshan 361511c2abaSLai Jiangshan /* can't move pfns which are higher than @z2 */ 362108bcc96SCody P Schafer if (end_pfn > zone_end_pfn(z2)) 363511c2abaSLai Jiangshan goto out_fail; 364834405c3SJiang Liu /* the move out part must be at the left most of @z2 */ 365511c2abaSLai Jiangshan if (start_pfn > z2->zone_start_pfn) 366511c2abaSLai Jiangshan goto out_fail; 367511c2abaSLai Jiangshan /* must included/overlap */ 368511c2abaSLai Jiangshan if (end_pfn <= z2->zone_start_pfn) 369511c2abaSLai Jiangshan goto out_fail; 370511c2abaSLai Jiangshan 371e455a9b9SLai Jiangshan /* use start_pfn for z1's start_pfn if z1 is empty */ 3728080fc03SXishi Qiu if (!zone_is_empty(z1)) 373e455a9b9SLai Jiangshan z1_start_pfn = z1->zone_start_pfn; 374e455a9b9SLai Jiangshan else 375e455a9b9SLai Jiangshan z1_start_pfn = start_pfn; 376e455a9b9SLai Jiangshan 377e455a9b9SLai Jiangshan resize_zone(z1, z1_start_pfn, end_pfn); 378108bcc96SCody P Schafer resize_zone(z2, end_pfn, zone_end_pfn(z2)); 379511c2abaSLai Jiangshan 380511c2abaSLai Jiangshan pgdat_resize_unlock(z1->zone_pgdat, &flags); 381511c2abaSLai Jiangshan 382511c2abaSLai Jiangshan fix_zone_id(z1, start_pfn, end_pfn); 383511c2abaSLai Jiangshan 384511c2abaSLai Jiangshan return 0; 385511c2abaSLai Jiangshan out_fail: 386511c2abaSLai Jiangshan pgdat_resize_unlock(z1->zone_pgdat, &flags); 387511c2abaSLai Jiangshan return -1; 388511c2abaSLai Jiangshan } 389511c2abaSLai Jiangshan 390e455a9b9SLai Jiangshan static int __meminit move_pfn_range_right(struct zone *z1, struct zone *z2, 391511c2abaSLai Jiangshan unsigned long start_pfn, unsigned long end_pfn) 392511c2abaSLai Jiangshan { 393e455a9b9SLai Jiangshan int ret; 394511c2abaSLai Jiangshan unsigned long flags; 395e455a9b9SLai Jiangshan unsigned long z2_end_pfn; 396e455a9b9SLai Jiangshan 39764dd1b29SCody P Schafer ret = ensure_zone_is_initialized(z2, start_pfn, end_pfn - start_pfn); 398e455a9b9SLai Jiangshan if (ret) 399e455a9b9SLai Jiangshan return ret; 400511c2abaSLai Jiangshan 401511c2abaSLai Jiangshan pgdat_resize_lock(z1->zone_pgdat, &flags); 402511c2abaSLai Jiangshan 403511c2abaSLai Jiangshan /* can't move pfns which are lower than @z1 */ 404511c2abaSLai Jiangshan if (z1->zone_start_pfn > start_pfn) 405511c2abaSLai Jiangshan goto out_fail; 406511c2abaSLai Jiangshan /* the move out part mast at the right most of @z1 */ 407108bcc96SCody P Schafer if (zone_end_pfn(z1) > end_pfn) 408511c2abaSLai Jiangshan goto out_fail; 409511c2abaSLai Jiangshan /* must included/overlap */ 410108bcc96SCody P Schafer if (start_pfn >= zone_end_pfn(z1)) 411511c2abaSLai Jiangshan goto out_fail; 412511c2abaSLai Jiangshan 413e455a9b9SLai Jiangshan /* use end_pfn for z2's end_pfn if z2 is empty */ 4148080fc03SXishi Qiu if (!zone_is_empty(z2)) 415108bcc96SCody P Schafer z2_end_pfn = zone_end_pfn(z2); 416e455a9b9SLai Jiangshan else 417e455a9b9SLai Jiangshan z2_end_pfn = end_pfn; 418e455a9b9SLai Jiangshan 419511c2abaSLai Jiangshan resize_zone(z1, z1->zone_start_pfn, start_pfn); 420e455a9b9SLai Jiangshan resize_zone(z2, start_pfn, z2_end_pfn); 421511c2abaSLai Jiangshan 422511c2abaSLai Jiangshan pgdat_resize_unlock(z1->zone_pgdat, &flags); 423511c2abaSLai Jiangshan 424511c2abaSLai Jiangshan fix_zone_id(z2, start_pfn, end_pfn); 425511c2abaSLai Jiangshan 426511c2abaSLai Jiangshan return 0; 427511c2abaSLai Jiangshan out_fail: 428511c2abaSLai Jiangshan pgdat_resize_unlock(z1->zone_pgdat, &flags); 429511c2abaSLai Jiangshan return -1; 430511c2abaSLai Jiangshan } 431511c2abaSLai Jiangshan 432f2765404SFabian Frederick static void __meminit grow_pgdat_span(struct pglist_data *pgdat, unsigned long start_pfn, 43376cdd58eSHeiko Carstens unsigned long end_pfn) 43476cdd58eSHeiko Carstens { 43583285c72SXishi Qiu unsigned long old_pgdat_end_pfn = pgdat_end_pfn(pgdat); 43676cdd58eSHeiko Carstens 437712cd386STang Chen if (!pgdat->node_spanned_pages || start_pfn < pgdat->node_start_pfn) 43876cdd58eSHeiko Carstens pgdat->node_start_pfn = start_pfn; 43976cdd58eSHeiko Carstens 44076cdd58eSHeiko Carstens pgdat->node_spanned_pages = max(old_pgdat_end_pfn, end_pfn) - 44176cdd58eSHeiko Carstens pgdat->node_start_pfn; 44276cdd58eSHeiko Carstens } 44376cdd58eSHeiko Carstens 44431168481SAl Viro static int __meminit __add_zone(struct zone *zone, unsigned long phys_start_pfn) 4453947be19SDave Hansen { 4463947be19SDave Hansen struct pglist_data *pgdat = zone->zone_pgdat; 4473947be19SDave Hansen int nr_pages = PAGES_PER_SECTION; 4483947be19SDave Hansen int nid = pgdat->node_id; 4493947be19SDave Hansen int zone_type; 450e298ff75SMel Gorman unsigned long flags, pfn; 45176cdd58eSHeiko Carstens int ret; 45276cdd58eSHeiko Carstens 45364dd1b29SCody P Schafer zone_type = zone - pgdat->node_zones; 45464dd1b29SCody P Schafer ret = ensure_zone_is_initialized(zone, phys_start_pfn, nr_pages); 45576cdd58eSHeiko Carstens if (ret) 45676cdd58eSHeiko Carstens return ret; 45764dd1b29SCody P Schafer 45876cdd58eSHeiko Carstens pgdat_resize_lock(zone->zone_pgdat, &flags); 45976cdd58eSHeiko Carstens grow_zone_span(zone, phys_start_pfn, phys_start_pfn + nr_pages); 46076cdd58eSHeiko Carstens grow_pgdat_span(zone->zone_pgdat, phys_start_pfn, 46176cdd58eSHeiko Carstens phys_start_pfn + nr_pages); 46276cdd58eSHeiko Carstens pgdat_resize_unlock(zone->zone_pgdat, &flags); 463a2f3aa02SDave Hansen memmap_init_zone(nr_pages, nid, zone_type, 464a2f3aa02SDave Hansen phys_start_pfn, MEMMAP_HOTPLUG); 465e298ff75SMel Gorman 466e298ff75SMel Gorman /* online_page_range is called later and expects pages reserved */ 467e298ff75SMel Gorman for (pfn = phys_start_pfn; pfn < phys_start_pfn + nr_pages; pfn++) { 468e298ff75SMel Gorman if (!pfn_valid(pfn)) 469e298ff75SMel Gorman continue; 470e298ff75SMel Gorman 471e298ff75SMel Gorman SetPageReserved(pfn_to_page(pfn)); 472e298ff75SMel Gorman } 473718127ccSYasunori Goto return 0; 4743947be19SDave Hansen } 4753947be19SDave Hansen 476c04fc586SGary Hade static int __meminit __add_section(int nid, struct zone *zone, 477c04fc586SGary Hade unsigned long phys_start_pfn) 4783947be19SDave Hansen { 4793947be19SDave Hansen int ret; 4803947be19SDave Hansen 481ebd15302SKAMEZAWA Hiroyuki if (pfn_valid(phys_start_pfn)) 482ebd15302SKAMEZAWA Hiroyuki return -EEXIST; 483ebd15302SKAMEZAWA Hiroyuki 48485b35feaSZhang Yanfei ret = sparse_add_one_section(zone, phys_start_pfn); 4853947be19SDave Hansen 4863947be19SDave Hansen if (ret < 0) 4873947be19SDave Hansen return ret; 4883947be19SDave Hansen 489718127ccSYasunori Goto ret = __add_zone(zone, phys_start_pfn); 490718127ccSYasunori Goto 491718127ccSYasunori Goto if (ret < 0) 492718127ccSYasunori Goto return ret; 493718127ccSYasunori Goto 494c04fc586SGary Hade return register_new_memory(nid, __pfn_to_section(phys_start_pfn)); 4953947be19SDave Hansen } 4963947be19SDave Hansen 4974edd7cefSDavid Rientjes /* 4984edd7cefSDavid Rientjes * Reasonably generic function for adding memory. It is 4994edd7cefSDavid Rientjes * expected that archs that support memory hotplug will 5004edd7cefSDavid Rientjes * call this function after deciding the zone to which to 5014edd7cefSDavid Rientjes * add the new pages. 5024edd7cefSDavid Rientjes */ 5034edd7cefSDavid Rientjes int __ref __add_pages(int nid, struct zone *zone, unsigned long phys_start_pfn, 5044edd7cefSDavid Rientjes unsigned long nr_pages) 5054edd7cefSDavid Rientjes { 5064edd7cefSDavid Rientjes unsigned long i; 5074edd7cefSDavid Rientjes int err = 0; 5084edd7cefSDavid Rientjes int start_sec, end_sec; 5094edd7cefSDavid Rientjes /* during initialize mem_map, align hot-added range to section */ 5104edd7cefSDavid Rientjes start_sec = pfn_to_section_nr(phys_start_pfn); 5114edd7cefSDavid Rientjes end_sec = pfn_to_section_nr(phys_start_pfn + nr_pages - 1); 5124edd7cefSDavid Rientjes 5134edd7cefSDavid Rientjes for (i = start_sec; i <= end_sec; i++) { 51419c07d5eSSheng Yong err = __add_section(nid, zone, section_nr_to_pfn(i)); 5154edd7cefSDavid Rientjes 5164edd7cefSDavid Rientjes /* 5174edd7cefSDavid Rientjes * EEXIST is finally dealt with by ioresource collision 5184edd7cefSDavid Rientjes * check. see add_memory() => register_memory_resource() 5194edd7cefSDavid Rientjes * Warning will be printed if there is collision. 5204edd7cefSDavid Rientjes */ 5214edd7cefSDavid Rientjes if (err && (err != -EEXIST)) 5224edd7cefSDavid Rientjes break; 5234edd7cefSDavid Rientjes err = 0; 5244edd7cefSDavid Rientjes } 525c435a390SZhu Guihua vmemmap_populate_print_last(); 5264edd7cefSDavid Rientjes 5274edd7cefSDavid Rientjes return err; 5284edd7cefSDavid Rientjes } 5294edd7cefSDavid Rientjes EXPORT_SYMBOL_GPL(__add_pages); 5304edd7cefSDavid Rientjes 5314edd7cefSDavid Rientjes #ifdef CONFIG_MEMORY_HOTREMOVE 532815121d2SYasuaki Ishimatsu /* find the smallest valid pfn in the range [start_pfn, end_pfn) */ 533815121d2SYasuaki Ishimatsu static int find_smallest_section_pfn(int nid, struct zone *zone, 534815121d2SYasuaki Ishimatsu unsigned long start_pfn, 535815121d2SYasuaki Ishimatsu unsigned long end_pfn) 536815121d2SYasuaki Ishimatsu { 537815121d2SYasuaki Ishimatsu struct mem_section *ms; 538815121d2SYasuaki Ishimatsu 539815121d2SYasuaki Ishimatsu for (; start_pfn < end_pfn; start_pfn += PAGES_PER_SECTION) { 540815121d2SYasuaki Ishimatsu ms = __pfn_to_section(start_pfn); 541815121d2SYasuaki Ishimatsu 542815121d2SYasuaki Ishimatsu if (unlikely(!valid_section(ms))) 543815121d2SYasuaki Ishimatsu continue; 544815121d2SYasuaki Ishimatsu 545815121d2SYasuaki Ishimatsu if (unlikely(pfn_to_nid(start_pfn) != nid)) 546815121d2SYasuaki Ishimatsu continue; 547815121d2SYasuaki Ishimatsu 548815121d2SYasuaki Ishimatsu if (zone && zone != page_zone(pfn_to_page(start_pfn))) 549815121d2SYasuaki Ishimatsu continue; 550815121d2SYasuaki Ishimatsu 551815121d2SYasuaki Ishimatsu return start_pfn; 552815121d2SYasuaki Ishimatsu } 553815121d2SYasuaki Ishimatsu 554815121d2SYasuaki Ishimatsu return 0; 555815121d2SYasuaki Ishimatsu } 556815121d2SYasuaki Ishimatsu 557815121d2SYasuaki Ishimatsu /* find the biggest valid pfn in the range [start_pfn, end_pfn). */ 558815121d2SYasuaki Ishimatsu static int find_biggest_section_pfn(int nid, struct zone *zone, 559815121d2SYasuaki Ishimatsu unsigned long start_pfn, 560815121d2SYasuaki Ishimatsu unsigned long end_pfn) 561815121d2SYasuaki Ishimatsu { 562815121d2SYasuaki Ishimatsu struct mem_section *ms; 563815121d2SYasuaki Ishimatsu unsigned long pfn; 564815121d2SYasuaki Ishimatsu 565815121d2SYasuaki Ishimatsu /* pfn is the end pfn of a memory section. */ 566815121d2SYasuaki Ishimatsu pfn = end_pfn - 1; 567815121d2SYasuaki Ishimatsu for (; pfn >= start_pfn; pfn -= PAGES_PER_SECTION) { 568815121d2SYasuaki Ishimatsu ms = __pfn_to_section(pfn); 569815121d2SYasuaki Ishimatsu 570815121d2SYasuaki Ishimatsu if (unlikely(!valid_section(ms))) 571815121d2SYasuaki Ishimatsu continue; 572815121d2SYasuaki Ishimatsu 573815121d2SYasuaki Ishimatsu if (unlikely(pfn_to_nid(pfn) != nid)) 574815121d2SYasuaki Ishimatsu continue; 575815121d2SYasuaki Ishimatsu 576815121d2SYasuaki Ishimatsu if (zone && zone != page_zone(pfn_to_page(pfn))) 577815121d2SYasuaki Ishimatsu continue; 578815121d2SYasuaki Ishimatsu 579815121d2SYasuaki Ishimatsu return pfn; 580815121d2SYasuaki Ishimatsu } 581815121d2SYasuaki Ishimatsu 582815121d2SYasuaki Ishimatsu return 0; 583815121d2SYasuaki Ishimatsu } 584815121d2SYasuaki Ishimatsu 585815121d2SYasuaki Ishimatsu static void shrink_zone_span(struct zone *zone, unsigned long start_pfn, 586815121d2SYasuaki Ishimatsu unsigned long end_pfn) 587815121d2SYasuaki Ishimatsu { 588815121d2SYasuaki Ishimatsu unsigned long zone_start_pfn = zone->zone_start_pfn; 589c33bc315SXishi Qiu unsigned long z = zone_end_pfn(zone); /* zone_end_pfn namespace clash */ 590c33bc315SXishi Qiu unsigned long zone_end_pfn = z; 591815121d2SYasuaki Ishimatsu unsigned long pfn; 592815121d2SYasuaki Ishimatsu struct mem_section *ms; 593815121d2SYasuaki Ishimatsu int nid = zone_to_nid(zone); 594815121d2SYasuaki Ishimatsu 595815121d2SYasuaki Ishimatsu zone_span_writelock(zone); 596815121d2SYasuaki Ishimatsu if (zone_start_pfn == start_pfn) { 597815121d2SYasuaki Ishimatsu /* 598815121d2SYasuaki Ishimatsu * If the section is smallest section in the zone, it need 599815121d2SYasuaki Ishimatsu * shrink zone->zone_start_pfn and zone->zone_spanned_pages. 600815121d2SYasuaki Ishimatsu * In this case, we find second smallest valid mem_section 601815121d2SYasuaki Ishimatsu * for shrinking zone. 602815121d2SYasuaki Ishimatsu */ 603815121d2SYasuaki Ishimatsu pfn = find_smallest_section_pfn(nid, zone, end_pfn, 604815121d2SYasuaki Ishimatsu zone_end_pfn); 605815121d2SYasuaki Ishimatsu if (pfn) { 606815121d2SYasuaki Ishimatsu zone->zone_start_pfn = pfn; 607815121d2SYasuaki Ishimatsu zone->spanned_pages = zone_end_pfn - pfn; 608815121d2SYasuaki Ishimatsu } 609815121d2SYasuaki Ishimatsu } else if (zone_end_pfn == end_pfn) { 610815121d2SYasuaki Ishimatsu /* 611815121d2SYasuaki Ishimatsu * If the section is biggest section in the zone, it need 612815121d2SYasuaki Ishimatsu * shrink zone->spanned_pages. 613815121d2SYasuaki Ishimatsu * In this case, we find second biggest valid mem_section for 614815121d2SYasuaki Ishimatsu * shrinking zone. 615815121d2SYasuaki Ishimatsu */ 616815121d2SYasuaki Ishimatsu pfn = find_biggest_section_pfn(nid, zone, zone_start_pfn, 617815121d2SYasuaki Ishimatsu start_pfn); 618815121d2SYasuaki Ishimatsu if (pfn) 619815121d2SYasuaki Ishimatsu zone->spanned_pages = pfn - zone_start_pfn + 1; 620815121d2SYasuaki Ishimatsu } 621815121d2SYasuaki Ishimatsu 622815121d2SYasuaki Ishimatsu /* 623815121d2SYasuaki Ishimatsu * The section is not biggest or smallest mem_section in the zone, it 624815121d2SYasuaki Ishimatsu * only creates a hole in the zone. So in this case, we need not 625815121d2SYasuaki Ishimatsu * change the zone. But perhaps, the zone has only hole data. Thus 626815121d2SYasuaki Ishimatsu * it check the zone has only hole or not. 627815121d2SYasuaki Ishimatsu */ 628815121d2SYasuaki Ishimatsu pfn = zone_start_pfn; 629815121d2SYasuaki Ishimatsu for (; pfn < zone_end_pfn; pfn += PAGES_PER_SECTION) { 630815121d2SYasuaki Ishimatsu ms = __pfn_to_section(pfn); 631815121d2SYasuaki Ishimatsu 632815121d2SYasuaki Ishimatsu if (unlikely(!valid_section(ms))) 633815121d2SYasuaki Ishimatsu continue; 634815121d2SYasuaki Ishimatsu 635815121d2SYasuaki Ishimatsu if (page_zone(pfn_to_page(pfn)) != zone) 636815121d2SYasuaki Ishimatsu continue; 637815121d2SYasuaki Ishimatsu 638815121d2SYasuaki Ishimatsu /* If the section is current section, it continues the loop */ 639815121d2SYasuaki Ishimatsu if (start_pfn == pfn) 640815121d2SYasuaki Ishimatsu continue; 641815121d2SYasuaki Ishimatsu 642815121d2SYasuaki Ishimatsu /* If we find valid section, we have nothing to do */ 643815121d2SYasuaki Ishimatsu zone_span_writeunlock(zone); 644815121d2SYasuaki Ishimatsu return; 645815121d2SYasuaki Ishimatsu } 646815121d2SYasuaki Ishimatsu 647815121d2SYasuaki Ishimatsu /* The zone has no valid section */ 648815121d2SYasuaki Ishimatsu zone->zone_start_pfn = 0; 649815121d2SYasuaki Ishimatsu zone->spanned_pages = 0; 650815121d2SYasuaki Ishimatsu zone_span_writeunlock(zone); 651815121d2SYasuaki Ishimatsu } 652815121d2SYasuaki Ishimatsu 653815121d2SYasuaki Ishimatsu static void shrink_pgdat_span(struct pglist_data *pgdat, 654815121d2SYasuaki Ishimatsu unsigned long start_pfn, unsigned long end_pfn) 655815121d2SYasuaki Ishimatsu { 656815121d2SYasuaki Ishimatsu unsigned long pgdat_start_pfn = pgdat->node_start_pfn; 65783285c72SXishi Qiu unsigned long p = pgdat_end_pfn(pgdat); /* pgdat_end_pfn namespace clash */ 65883285c72SXishi Qiu unsigned long pgdat_end_pfn = p; 659815121d2SYasuaki Ishimatsu unsigned long pfn; 660815121d2SYasuaki Ishimatsu struct mem_section *ms; 661815121d2SYasuaki Ishimatsu int nid = pgdat->node_id; 662815121d2SYasuaki Ishimatsu 663815121d2SYasuaki Ishimatsu if (pgdat_start_pfn == start_pfn) { 664815121d2SYasuaki Ishimatsu /* 665815121d2SYasuaki Ishimatsu * If the section is smallest section in the pgdat, it need 666815121d2SYasuaki Ishimatsu * shrink pgdat->node_start_pfn and pgdat->node_spanned_pages. 667815121d2SYasuaki Ishimatsu * In this case, we find second smallest valid mem_section 668815121d2SYasuaki Ishimatsu * for shrinking zone. 669815121d2SYasuaki Ishimatsu */ 670815121d2SYasuaki Ishimatsu pfn = find_smallest_section_pfn(nid, NULL, end_pfn, 671815121d2SYasuaki Ishimatsu pgdat_end_pfn); 672815121d2SYasuaki Ishimatsu if (pfn) { 673815121d2SYasuaki Ishimatsu pgdat->node_start_pfn = pfn; 674815121d2SYasuaki Ishimatsu pgdat->node_spanned_pages = pgdat_end_pfn - pfn; 675815121d2SYasuaki Ishimatsu } 676815121d2SYasuaki Ishimatsu } else if (pgdat_end_pfn == end_pfn) { 677815121d2SYasuaki Ishimatsu /* 678815121d2SYasuaki Ishimatsu * If the section is biggest section in the pgdat, it need 679815121d2SYasuaki Ishimatsu * shrink pgdat->node_spanned_pages. 680815121d2SYasuaki Ishimatsu * In this case, we find second biggest valid mem_section for 681815121d2SYasuaki Ishimatsu * shrinking zone. 682815121d2SYasuaki Ishimatsu */ 683815121d2SYasuaki Ishimatsu pfn = find_biggest_section_pfn(nid, NULL, pgdat_start_pfn, 684815121d2SYasuaki Ishimatsu start_pfn); 685815121d2SYasuaki Ishimatsu if (pfn) 686815121d2SYasuaki Ishimatsu pgdat->node_spanned_pages = pfn - pgdat_start_pfn + 1; 687815121d2SYasuaki Ishimatsu } 688815121d2SYasuaki Ishimatsu 689815121d2SYasuaki Ishimatsu /* 690815121d2SYasuaki Ishimatsu * If the section is not biggest or smallest mem_section in the pgdat, 691815121d2SYasuaki Ishimatsu * it only creates a hole in the pgdat. So in this case, we need not 692815121d2SYasuaki Ishimatsu * change the pgdat. 693815121d2SYasuaki Ishimatsu * But perhaps, the pgdat has only hole data. Thus it check the pgdat 694815121d2SYasuaki Ishimatsu * has only hole or not. 695815121d2SYasuaki Ishimatsu */ 696815121d2SYasuaki Ishimatsu pfn = pgdat_start_pfn; 697815121d2SYasuaki Ishimatsu for (; pfn < pgdat_end_pfn; pfn += PAGES_PER_SECTION) { 698815121d2SYasuaki Ishimatsu ms = __pfn_to_section(pfn); 699815121d2SYasuaki Ishimatsu 700815121d2SYasuaki Ishimatsu if (unlikely(!valid_section(ms))) 701815121d2SYasuaki Ishimatsu continue; 702815121d2SYasuaki Ishimatsu 703815121d2SYasuaki Ishimatsu if (pfn_to_nid(pfn) != nid) 704815121d2SYasuaki Ishimatsu continue; 705815121d2SYasuaki Ishimatsu 706815121d2SYasuaki Ishimatsu /* If the section is current section, it continues the loop */ 707815121d2SYasuaki Ishimatsu if (start_pfn == pfn) 708815121d2SYasuaki Ishimatsu continue; 709815121d2SYasuaki Ishimatsu 710815121d2SYasuaki Ishimatsu /* If we find valid section, we have nothing to do */ 711815121d2SYasuaki Ishimatsu return; 712815121d2SYasuaki Ishimatsu } 713815121d2SYasuaki Ishimatsu 714815121d2SYasuaki Ishimatsu /* The pgdat has no valid section */ 715815121d2SYasuaki Ishimatsu pgdat->node_start_pfn = 0; 716815121d2SYasuaki Ishimatsu pgdat->node_spanned_pages = 0; 717815121d2SYasuaki Ishimatsu } 718815121d2SYasuaki Ishimatsu 719815121d2SYasuaki Ishimatsu static void __remove_zone(struct zone *zone, unsigned long start_pfn) 720815121d2SYasuaki Ishimatsu { 721815121d2SYasuaki Ishimatsu struct pglist_data *pgdat = zone->zone_pgdat; 722815121d2SYasuaki Ishimatsu int nr_pages = PAGES_PER_SECTION; 723815121d2SYasuaki Ishimatsu int zone_type; 724815121d2SYasuaki Ishimatsu unsigned long flags; 725815121d2SYasuaki Ishimatsu 726815121d2SYasuaki Ishimatsu zone_type = zone - pgdat->node_zones; 727815121d2SYasuaki Ishimatsu 728815121d2SYasuaki Ishimatsu pgdat_resize_lock(zone->zone_pgdat, &flags); 729815121d2SYasuaki Ishimatsu shrink_zone_span(zone, start_pfn, start_pfn + nr_pages); 730815121d2SYasuaki Ishimatsu shrink_pgdat_span(pgdat, start_pfn, start_pfn + nr_pages); 731815121d2SYasuaki Ishimatsu pgdat_resize_unlock(zone->zone_pgdat, &flags); 732815121d2SYasuaki Ishimatsu } 733815121d2SYasuaki Ishimatsu 734ea01ea93SBadari Pulavarty static int __remove_section(struct zone *zone, struct mem_section *ms) 735ea01ea93SBadari Pulavarty { 736815121d2SYasuaki Ishimatsu unsigned long start_pfn; 737815121d2SYasuaki Ishimatsu int scn_nr; 738ea01ea93SBadari Pulavarty int ret = -EINVAL; 739ea01ea93SBadari Pulavarty 740ea01ea93SBadari Pulavarty if (!valid_section(ms)) 741ea01ea93SBadari Pulavarty return ret; 742ea01ea93SBadari Pulavarty 743ea01ea93SBadari Pulavarty ret = unregister_memory_section(ms); 744ea01ea93SBadari Pulavarty if (ret) 745ea01ea93SBadari Pulavarty return ret; 746ea01ea93SBadari Pulavarty 747815121d2SYasuaki Ishimatsu scn_nr = __section_nr(ms); 748815121d2SYasuaki Ishimatsu start_pfn = section_nr_to_pfn(scn_nr); 749815121d2SYasuaki Ishimatsu __remove_zone(zone, start_pfn); 750815121d2SYasuaki Ishimatsu 751ea01ea93SBadari Pulavarty sparse_remove_one_section(zone, ms); 752ea01ea93SBadari Pulavarty return 0; 753ea01ea93SBadari Pulavarty } 754ea01ea93SBadari Pulavarty 755ea01ea93SBadari Pulavarty /** 756ea01ea93SBadari Pulavarty * __remove_pages() - remove sections of pages from a zone 757ea01ea93SBadari Pulavarty * @zone: zone from which pages need to be removed 758ea01ea93SBadari Pulavarty * @phys_start_pfn: starting pageframe (must be aligned to start of a section) 759ea01ea93SBadari Pulavarty * @nr_pages: number of pages to remove (must be multiple of section size) 760ea01ea93SBadari Pulavarty * 761ea01ea93SBadari Pulavarty * Generic helper function to remove section mappings and sysfs entries 762ea01ea93SBadari Pulavarty * for the section of the memory we are removing. Caller needs to make 763ea01ea93SBadari Pulavarty * sure that pages are marked reserved and zones are adjust properly by 764ea01ea93SBadari Pulavarty * calling offline_pages(). 765ea01ea93SBadari Pulavarty */ 766ea01ea93SBadari Pulavarty int __remove_pages(struct zone *zone, unsigned long phys_start_pfn, 767ea01ea93SBadari Pulavarty unsigned long nr_pages) 768ea01ea93SBadari Pulavarty { 769fe74ebb1SToshi Kani unsigned long i; 770ea01ea93SBadari Pulavarty int sections_to_remove; 771fe74ebb1SToshi Kani resource_size_t start, size; 772fe74ebb1SToshi Kani int ret = 0; 773ea01ea93SBadari Pulavarty 774ea01ea93SBadari Pulavarty /* 775ea01ea93SBadari Pulavarty * We can only remove entire sections 776ea01ea93SBadari Pulavarty */ 777ea01ea93SBadari Pulavarty BUG_ON(phys_start_pfn & ~PAGE_SECTION_MASK); 778ea01ea93SBadari Pulavarty BUG_ON(nr_pages % PAGES_PER_SECTION); 779ea01ea93SBadari Pulavarty 780fe74ebb1SToshi Kani start = phys_start_pfn << PAGE_SHIFT; 781fe74ebb1SToshi Kani size = nr_pages * PAGE_SIZE; 782033fbae9SDan Williams 783033fbae9SDan Williams /* in the ZONE_DEVICE case device driver owns the memory region */ 784033fbae9SDan Williams if (!is_dev_zone(zone)) 785fe74ebb1SToshi Kani ret = release_mem_region_adjustable(&iomem_resource, start, size); 786348f9f05SRandy Dunlap if (ret) { 787348f9f05SRandy Dunlap resource_size_t endres = start + size - 1; 788348f9f05SRandy Dunlap 789348f9f05SRandy Dunlap pr_warn("Unable to release resource <%pa-%pa> (%d)\n", 790348f9f05SRandy Dunlap &start, &endres, ret); 791348f9f05SRandy Dunlap } 792d760afd4SYasuaki Ishimatsu 793ea01ea93SBadari Pulavarty sections_to_remove = nr_pages / PAGES_PER_SECTION; 794ea01ea93SBadari Pulavarty for (i = 0; i < sections_to_remove; i++) { 795ea01ea93SBadari Pulavarty unsigned long pfn = phys_start_pfn + i*PAGES_PER_SECTION; 796ea01ea93SBadari Pulavarty ret = __remove_section(zone, __pfn_to_section(pfn)); 797ea01ea93SBadari Pulavarty if (ret) 798ea01ea93SBadari Pulavarty break; 799ea01ea93SBadari Pulavarty } 800ea01ea93SBadari Pulavarty return ret; 801ea01ea93SBadari Pulavarty } 802ea01ea93SBadari Pulavarty EXPORT_SYMBOL_GPL(__remove_pages); 8034edd7cefSDavid Rientjes #endif /* CONFIG_MEMORY_HOTREMOVE */ 804ea01ea93SBadari Pulavarty 8059d0ad8caSDaniel Kiper int set_online_page_callback(online_page_callback_t callback) 8069d0ad8caSDaniel Kiper { 8079d0ad8caSDaniel Kiper int rc = -EINVAL; 8089d0ad8caSDaniel Kiper 809bfc8c901SVladimir Davydov get_online_mems(); 810bfc8c901SVladimir Davydov mutex_lock(&online_page_callback_lock); 8119d0ad8caSDaniel Kiper 8129d0ad8caSDaniel Kiper if (online_page_callback == generic_online_page) { 8139d0ad8caSDaniel Kiper online_page_callback = callback; 8149d0ad8caSDaniel Kiper rc = 0; 8159d0ad8caSDaniel Kiper } 8169d0ad8caSDaniel Kiper 817bfc8c901SVladimir Davydov mutex_unlock(&online_page_callback_lock); 818bfc8c901SVladimir Davydov put_online_mems(); 8199d0ad8caSDaniel Kiper 8209d0ad8caSDaniel Kiper return rc; 8219d0ad8caSDaniel Kiper } 8229d0ad8caSDaniel Kiper EXPORT_SYMBOL_GPL(set_online_page_callback); 8239d0ad8caSDaniel Kiper 8249d0ad8caSDaniel Kiper int restore_online_page_callback(online_page_callback_t callback) 8259d0ad8caSDaniel Kiper { 8269d0ad8caSDaniel Kiper int rc = -EINVAL; 8279d0ad8caSDaniel Kiper 828bfc8c901SVladimir Davydov get_online_mems(); 829bfc8c901SVladimir Davydov mutex_lock(&online_page_callback_lock); 8309d0ad8caSDaniel Kiper 8319d0ad8caSDaniel Kiper if (online_page_callback == callback) { 8329d0ad8caSDaniel Kiper online_page_callback = generic_online_page; 8339d0ad8caSDaniel Kiper rc = 0; 8349d0ad8caSDaniel Kiper } 8359d0ad8caSDaniel Kiper 836bfc8c901SVladimir Davydov mutex_unlock(&online_page_callback_lock); 837bfc8c901SVladimir Davydov put_online_mems(); 8389d0ad8caSDaniel Kiper 8399d0ad8caSDaniel Kiper return rc; 8409d0ad8caSDaniel Kiper } 8419d0ad8caSDaniel Kiper EXPORT_SYMBOL_GPL(restore_online_page_callback); 8429d0ad8caSDaniel Kiper 8439d0ad8caSDaniel Kiper void __online_page_set_limits(struct page *page) 844180c06efSJeremy Fitzhardinge { 8459d0ad8caSDaniel Kiper } 8469d0ad8caSDaniel Kiper EXPORT_SYMBOL_GPL(__online_page_set_limits); 8479d0ad8caSDaniel Kiper 8489d0ad8caSDaniel Kiper void __online_page_increment_counters(struct page *page) 8499d0ad8caSDaniel Kiper { 8503dcc0571SJiang Liu adjust_managed_page_count(page, 1); 8519d0ad8caSDaniel Kiper } 8529d0ad8caSDaniel Kiper EXPORT_SYMBOL_GPL(__online_page_increment_counters); 853180c06efSJeremy Fitzhardinge 8549d0ad8caSDaniel Kiper void __online_page_free(struct page *page) 8559d0ad8caSDaniel Kiper { 8563dcc0571SJiang Liu __free_reserved_page(page); 857180c06efSJeremy Fitzhardinge } 8589d0ad8caSDaniel Kiper EXPORT_SYMBOL_GPL(__online_page_free); 8599d0ad8caSDaniel Kiper 8609d0ad8caSDaniel Kiper static void generic_online_page(struct page *page) 8619d0ad8caSDaniel Kiper { 8629d0ad8caSDaniel Kiper __online_page_set_limits(page); 8639d0ad8caSDaniel Kiper __online_page_increment_counters(page); 8649d0ad8caSDaniel Kiper __online_page_free(page); 8659d0ad8caSDaniel Kiper } 866180c06efSJeremy Fitzhardinge 86775884fb1SKAMEZAWA Hiroyuki static int online_pages_range(unsigned long start_pfn, unsigned long nr_pages, 86875884fb1SKAMEZAWA Hiroyuki void *arg) 8693947be19SDave Hansen { 8703947be19SDave Hansen unsigned long i; 87175884fb1SKAMEZAWA Hiroyuki unsigned long onlined_pages = *(unsigned long *)arg; 87275884fb1SKAMEZAWA Hiroyuki struct page *page; 87375884fb1SKAMEZAWA Hiroyuki if (PageReserved(pfn_to_page(start_pfn))) 87475884fb1SKAMEZAWA Hiroyuki for (i = 0; i < nr_pages; i++) { 87575884fb1SKAMEZAWA Hiroyuki page = pfn_to_page(start_pfn + i); 8769d0ad8caSDaniel Kiper (*online_page_callback)(page); 87775884fb1SKAMEZAWA Hiroyuki onlined_pages++; 87875884fb1SKAMEZAWA Hiroyuki } 87975884fb1SKAMEZAWA Hiroyuki *(unsigned long *)arg = onlined_pages; 88075884fb1SKAMEZAWA Hiroyuki return 0; 88175884fb1SKAMEZAWA Hiroyuki } 88275884fb1SKAMEZAWA Hiroyuki 88309285af7SLai Jiangshan #ifdef CONFIG_MOVABLE_NODE 88479a4dcefSTang Chen /* 88579a4dcefSTang Chen * When CONFIG_MOVABLE_NODE, we permit onlining of a node which doesn't have 88679a4dcefSTang Chen * normal memory. 88779a4dcefSTang Chen */ 88809285af7SLai Jiangshan static bool can_online_high_movable(struct zone *zone) 88909285af7SLai Jiangshan { 89009285af7SLai Jiangshan return true; 89109285af7SLai Jiangshan } 89279a4dcefSTang Chen #else /* CONFIG_MOVABLE_NODE */ 89374d42d8fSLai Jiangshan /* ensure every online node has NORMAL memory */ 89474d42d8fSLai Jiangshan static bool can_online_high_movable(struct zone *zone) 89574d42d8fSLai Jiangshan { 89674d42d8fSLai Jiangshan return node_state(zone_to_nid(zone), N_NORMAL_MEMORY); 89774d42d8fSLai Jiangshan } 89879a4dcefSTang Chen #endif /* CONFIG_MOVABLE_NODE */ 89974d42d8fSLai Jiangshan 900d9713679SLai Jiangshan /* check which state of node_states will be changed when online memory */ 901d9713679SLai Jiangshan static void node_states_check_changes_online(unsigned long nr_pages, 902d9713679SLai Jiangshan struct zone *zone, struct memory_notify *arg) 903d9713679SLai Jiangshan { 904d9713679SLai Jiangshan int nid = zone_to_nid(zone); 905d9713679SLai Jiangshan enum zone_type zone_last = ZONE_NORMAL; 906d9713679SLai Jiangshan 907d9713679SLai Jiangshan /* 9086715ddf9SLai Jiangshan * If we have HIGHMEM or movable node, node_states[N_NORMAL_MEMORY] 9096715ddf9SLai Jiangshan * contains nodes which have zones of 0...ZONE_NORMAL, 9106715ddf9SLai Jiangshan * set zone_last to ZONE_NORMAL. 911d9713679SLai Jiangshan * 9126715ddf9SLai Jiangshan * If we don't have HIGHMEM nor movable node, 9136715ddf9SLai Jiangshan * node_states[N_NORMAL_MEMORY] contains nodes which have zones of 9146715ddf9SLai Jiangshan * 0...ZONE_MOVABLE, set zone_last to ZONE_MOVABLE. 915d9713679SLai Jiangshan */ 9166715ddf9SLai Jiangshan if (N_MEMORY == N_NORMAL_MEMORY) 917d9713679SLai Jiangshan zone_last = ZONE_MOVABLE; 918d9713679SLai Jiangshan 919d9713679SLai Jiangshan /* 920d9713679SLai Jiangshan * if the memory to be online is in a zone of 0...zone_last, and 921d9713679SLai Jiangshan * the zones of 0...zone_last don't have memory before online, we will 922d9713679SLai Jiangshan * need to set the node to node_states[N_NORMAL_MEMORY] after 923d9713679SLai Jiangshan * the memory is online. 924d9713679SLai Jiangshan */ 925d9713679SLai Jiangshan if (zone_idx(zone) <= zone_last && !node_state(nid, N_NORMAL_MEMORY)) 926d9713679SLai Jiangshan arg->status_change_nid_normal = nid; 927d9713679SLai Jiangshan else 928d9713679SLai Jiangshan arg->status_change_nid_normal = -1; 929d9713679SLai Jiangshan 9306715ddf9SLai Jiangshan #ifdef CONFIG_HIGHMEM 9316715ddf9SLai Jiangshan /* 9326715ddf9SLai Jiangshan * If we have movable node, node_states[N_HIGH_MEMORY] 9336715ddf9SLai Jiangshan * contains nodes which have zones of 0...ZONE_HIGHMEM, 9346715ddf9SLai Jiangshan * set zone_last to ZONE_HIGHMEM. 9356715ddf9SLai Jiangshan * 9366715ddf9SLai Jiangshan * If we don't have movable node, node_states[N_NORMAL_MEMORY] 9376715ddf9SLai Jiangshan * contains nodes which have zones of 0...ZONE_MOVABLE, 9386715ddf9SLai Jiangshan * set zone_last to ZONE_MOVABLE. 9396715ddf9SLai Jiangshan */ 9406715ddf9SLai Jiangshan zone_last = ZONE_HIGHMEM; 9416715ddf9SLai Jiangshan if (N_MEMORY == N_HIGH_MEMORY) 9426715ddf9SLai Jiangshan zone_last = ZONE_MOVABLE; 9436715ddf9SLai Jiangshan 9446715ddf9SLai Jiangshan if (zone_idx(zone) <= zone_last && !node_state(nid, N_HIGH_MEMORY)) 9456715ddf9SLai Jiangshan arg->status_change_nid_high = nid; 9466715ddf9SLai Jiangshan else 9476715ddf9SLai Jiangshan arg->status_change_nid_high = -1; 9486715ddf9SLai Jiangshan #else 9496715ddf9SLai Jiangshan arg->status_change_nid_high = arg->status_change_nid_normal; 9506715ddf9SLai Jiangshan #endif 9516715ddf9SLai Jiangshan 952d9713679SLai Jiangshan /* 953d9713679SLai Jiangshan * if the node don't have memory befor online, we will need to 9546715ddf9SLai Jiangshan * set the node to node_states[N_MEMORY] after the memory 955d9713679SLai Jiangshan * is online. 956d9713679SLai Jiangshan */ 9576715ddf9SLai Jiangshan if (!node_state(nid, N_MEMORY)) 958d9713679SLai Jiangshan arg->status_change_nid = nid; 959d9713679SLai Jiangshan else 960d9713679SLai Jiangshan arg->status_change_nid = -1; 961d9713679SLai Jiangshan } 962d9713679SLai Jiangshan 963d9713679SLai Jiangshan static void node_states_set_node(int node, struct memory_notify *arg) 964d9713679SLai Jiangshan { 965d9713679SLai Jiangshan if (arg->status_change_nid_normal >= 0) 966d9713679SLai Jiangshan node_set_state(node, N_NORMAL_MEMORY); 967d9713679SLai Jiangshan 9686715ddf9SLai Jiangshan if (arg->status_change_nid_high >= 0) 969d9713679SLai Jiangshan node_set_state(node, N_HIGH_MEMORY); 9706715ddf9SLai Jiangshan 9716715ddf9SLai Jiangshan node_set_state(node, N_MEMORY); 972d9713679SLai Jiangshan } 973d9713679SLai Jiangshan 97475884fb1SKAMEZAWA Hiroyuki 97530467e0bSDavid Rientjes /* Must be protected by mem_hotplug_begin() */ 976511c2abaSLai Jiangshan int __ref online_pages(unsigned long pfn, unsigned long nr_pages, int online_type) 97775884fb1SKAMEZAWA Hiroyuki { 978aa47228aSCody P Schafer unsigned long flags; 9793947be19SDave Hansen unsigned long onlined_pages = 0; 9803947be19SDave Hansen struct zone *zone; 9816811378eSYasunori Goto int need_zonelists_rebuild = 0; 9827b78d335SYasunori Goto int nid; 9837b78d335SYasunori Goto int ret; 9847b78d335SYasunori Goto struct memory_notify arg; 9853947be19SDave Hansen 986d9713679SLai Jiangshan /* 987d9713679SLai Jiangshan * This doesn't need a lock to do pfn_to_page(). 988d9713679SLai Jiangshan * The section can't be removed here because of the 989d9713679SLai Jiangshan * memory_block->state_mutex. 990d9713679SLai Jiangshan */ 991d9713679SLai Jiangshan zone = page_zone(pfn_to_page(pfn)); 992d9713679SLai Jiangshan 9934f7c6b49STang Chen if ((zone_idx(zone) > ZONE_NORMAL || 9944f7c6b49STang Chen online_type == MMOP_ONLINE_MOVABLE) && 995bfc8c901SVladimir Davydov !can_online_high_movable(zone)) 99630467e0bSDavid Rientjes return -EINVAL; 99774d42d8fSLai Jiangshan 9984f7c6b49STang Chen if (online_type == MMOP_ONLINE_KERNEL && 9994f7c6b49STang Chen zone_idx(zone) == ZONE_MOVABLE) { 1000bfc8c901SVladimir Davydov if (move_pfn_range_left(zone - 1, zone, pfn, pfn + nr_pages)) 100130467e0bSDavid Rientjes return -EINVAL; 1002511c2abaSLai Jiangshan } 10034f7c6b49STang Chen if (online_type == MMOP_ONLINE_MOVABLE && 10044f7c6b49STang Chen zone_idx(zone) == ZONE_MOVABLE - 1) { 1005bfc8c901SVladimir Davydov if (move_pfn_range_right(zone, zone + 1, pfn, pfn + nr_pages)) 100630467e0bSDavid Rientjes return -EINVAL; 1007511c2abaSLai Jiangshan } 1008511c2abaSLai Jiangshan 1009511c2abaSLai Jiangshan /* Previous code may changed the zone of the pfn range */ 1010511c2abaSLai Jiangshan zone = page_zone(pfn_to_page(pfn)); 1011511c2abaSLai Jiangshan 10127b78d335SYasunori Goto arg.start_pfn = pfn; 10137b78d335SYasunori Goto arg.nr_pages = nr_pages; 1014d9713679SLai Jiangshan node_states_check_changes_online(nr_pages, zone, &arg); 10157b78d335SYasunori Goto 10169c2606b7SXishi Qiu nid = pfn_to_nid(pfn); 10177b78d335SYasunori Goto 10187b78d335SYasunori Goto ret = memory_notify(MEM_GOING_ONLINE, &arg); 10197b78d335SYasunori Goto ret = notifier_to_errno(ret); 10207b78d335SYasunori Goto if (ret) { 10217b78d335SYasunori Goto memory_notify(MEM_CANCEL_ONLINE, &arg); 102230467e0bSDavid Rientjes return ret; 10237b78d335SYasunori Goto } 10243947be19SDave Hansen /* 10256811378eSYasunori Goto * If this zone is not populated, then it is not in zonelist. 10266811378eSYasunori Goto * This means the page allocator ignores this zone. 10276811378eSYasunori Goto * So, zonelist must be updated after online. 10286811378eSYasunori Goto */ 10294eaf3f64SHaicheng Li mutex_lock(&zonelists_mutex); 10306dcd73d7SWen Congyang if (!populated_zone(zone)) { 10316811378eSYasunori Goto need_zonelists_rebuild = 1; 10326dcd73d7SWen Congyang build_all_zonelists(NULL, zone); 10336dcd73d7SWen Congyang } 10346811378eSYasunori Goto 1035908eedc6SKAMEZAWA Hiroyuki ret = walk_system_ram_range(pfn, nr_pages, &onlined_pages, 103675884fb1SKAMEZAWA Hiroyuki online_pages_range); 1037fd8a4221SGeoff Levand if (ret) { 10386dcd73d7SWen Congyang if (need_zonelists_rebuild) 10396dcd73d7SWen Congyang zone_pcp_reset(zone); 10404eaf3f64SHaicheng Li mutex_unlock(&zonelists_mutex); 1041a62e2f4fSBjorn Helgaas printk(KERN_DEBUG "online_pages [mem %#010llx-%#010llx] failed\n", 1042a62e2f4fSBjorn Helgaas (unsigned long long) pfn << PAGE_SHIFT, 1043a62e2f4fSBjorn Helgaas (((unsigned long long) pfn + nr_pages) 1044a62e2f4fSBjorn Helgaas << PAGE_SHIFT) - 1); 1045fd8a4221SGeoff Levand memory_notify(MEM_CANCEL_ONLINE, &arg); 104630467e0bSDavid Rientjes return ret; 1047fd8a4221SGeoff Levand } 1048fd8a4221SGeoff Levand 10493947be19SDave Hansen zone->present_pages += onlined_pages; 1050aa47228aSCody P Schafer 1051aa47228aSCody P Schafer pgdat_resize_lock(zone->zone_pgdat, &flags); 1052f2937be5SYasunori Goto zone->zone_pgdat->node_present_pages += onlined_pages; 1053aa47228aSCody P Schafer pgdat_resize_unlock(zone->zone_pgdat, &flags); 1054aa47228aSCody P Schafer 105508dff7b7SJiang Liu if (onlined_pages) { 1056d9713679SLai Jiangshan node_states_set_node(zone_to_nid(zone), &arg); 10571f522509SHaicheng Li if (need_zonelists_rebuild) 10586dcd73d7SWen Congyang build_all_zonelists(NULL, NULL); 10591f522509SHaicheng Li else 1060112067f0SShaohua Li zone_pcp_update(zone); 106108dff7b7SJiang Liu } 10621f522509SHaicheng Li 10634eaf3f64SHaicheng Li mutex_unlock(&zonelists_mutex); 10641b79acc9SKOSAKI Motohiro 10651b79acc9SKOSAKI Motohiro init_per_zone_wmark_min(); 10661b79acc9SKOSAKI Motohiro 106708dff7b7SJiang Liu if (onlined_pages) 10687ea1530aSChristoph Lameter kswapd_run(zone_to_nid(zone)); 106961b13993SDave Hansen 10705a4d4361SKAMEZAWA Hiroyuki vm_total_pages = nr_free_pagecache_pages(); 10712f7f24ecSKent Liu 10722d1d43f6SChandra Seetharaman writeback_set_ratelimit(); 10737b78d335SYasunori Goto 10747b78d335SYasunori Goto if (onlined_pages) 10757b78d335SYasunori Goto memory_notify(MEM_ONLINE, &arg); 107630467e0bSDavid Rientjes return 0; 10773947be19SDave Hansen } 107853947027SKeith Mannthey #endif /* CONFIG_MEMORY_HOTPLUG_SPARSE */ 1079bc02af93SYasunori Goto 10800bd85420STang Chen static void reset_node_present_pages(pg_data_t *pgdat) 10810bd85420STang Chen { 10820bd85420STang Chen struct zone *z; 10830bd85420STang Chen 10840bd85420STang Chen for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++) 10850bd85420STang Chen z->present_pages = 0; 10860bd85420STang Chen 10870bd85420STang Chen pgdat->node_present_pages = 0; 10880bd85420STang Chen } 10890bd85420STang Chen 1090e1319331SHidetoshi Seto /* we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG */ 1091e1319331SHidetoshi Seto static pg_data_t __ref *hotadd_new_pgdat(int nid, u64 start) 10929af3c2deSYasunori Goto { 10939af3c2deSYasunori Goto struct pglist_data *pgdat; 10949af3c2deSYasunori Goto unsigned long zones_size[MAX_NR_ZONES] = {0}; 10959af3c2deSYasunori Goto unsigned long zholes_size[MAX_NR_ZONES] = {0}; 1096c8e861a5SFabian Frederick unsigned long start_pfn = PFN_DOWN(start); 10979af3c2deSYasunori Goto 1098a1e565aaSTang Chen pgdat = NODE_DATA(nid); 1099a1e565aaSTang Chen if (!pgdat) { 11009af3c2deSYasunori Goto pgdat = arch_alloc_nodedata(nid); 11019af3c2deSYasunori Goto if (!pgdat) 11029af3c2deSYasunori Goto return NULL; 11039af3c2deSYasunori Goto 11049af3c2deSYasunori Goto arch_refresh_nodedata(nid, pgdat); 1105b0dc3a34SGu Zheng } else { 1106b0dc3a34SGu Zheng /* Reset the nr_zones and classzone_idx to 0 before reuse */ 1107b0dc3a34SGu Zheng pgdat->nr_zones = 0; 1108b0dc3a34SGu Zheng pgdat->classzone_idx = 0; 1109a1e565aaSTang Chen } 11109af3c2deSYasunori Goto 11119af3c2deSYasunori Goto /* we can use NODE_DATA(nid) from here */ 11129af3c2deSYasunori Goto 11139af3c2deSYasunori Goto /* init node's zones as empty zones, we don't have any present pages.*/ 11149109fb7bSJohannes Weiner free_area_init_node(nid, zones_size, start_pfn, zholes_size); 11159af3c2deSYasunori Goto 1116959ecc48SKAMEZAWA Hiroyuki /* 1117959ecc48SKAMEZAWA Hiroyuki * The node we allocated has no zone fallback lists. For avoiding 1118959ecc48SKAMEZAWA Hiroyuki * to access not-initialized zonelist, build here. 1119959ecc48SKAMEZAWA Hiroyuki */ 1120f957db4fSDavid Rientjes mutex_lock(&zonelists_mutex); 11219adb62a5SJiang Liu build_all_zonelists(pgdat, NULL); 1122f957db4fSDavid Rientjes mutex_unlock(&zonelists_mutex); 1123959ecc48SKAMEZAWA Hiroyuki 1124f784a3f1STang Chen /* 1125f784a3f1STang Chen * zone->managed_pages is set to an approximate value in 1126f784a3f1STang Chen * free_area_init_core(), which will cause 1127f784a3f1STang Chen * /sys/device/system/node/nodeX/meminfo has wrong data. 1128f784a3f1STang Chen * So reset it to 0 before any memory is onlined. 1129f784a3f1STang Chen */ 1130f784a3f1STang Chen reset_node_managed_pages(pgdat); 1131f784a3f1STang Chen 11320bd85420STang Chen /* 11330bd85420STang Chen * When memory is hot-added, all the memory is in offline state. So 11340bd85420STang Chen * clear all zones' present_pages because they will be updated in 11350bd85420STang Chen * online_pages() and offline_pages(). 11360bd85420STang Chen */ 11370bd85420STang Chen reset_node_present_pages(pgdat); 11380bd85420STang Chen 11399af3c2deSYasunori Goto return pgdat; 11409af3c2deSYasunori Goto } 11419af3c2deSYasunori Goto 11429af3c2deSYasunori Goto static void rollback_node_hotadd(int nid, pg_data_t *pgdat) 11439af3c2deSYasunori Goto { 11449af3c2deSYasunori Goto arch_refresh_nodedata(nid, NULL); 11459af3c2deSYasunori Goto arch_free_nodedata(pgdat); 11469af3c2deSYasunori Goto return; 11479af3c2deSYasunori Goto } 11489af3c2deSYasunori Goto 11490a547039SKAMEZAWA Hiroyuki 115001b0f197SToshi Kani /** 115101b0f197SToshi Kani * try_online_node - online a node if offlined 115201b0f197SToshi Kani * 1153cf23422bSminskey guo * called by cpu_up() to online a node without onlined memory. 1154cf23422bSminskey guo */ 115501b0f197SToshi Kani int try_online_node(int nid) 1156cf23422bSminskey guo { 1157cf23422bSminskey guo pg_data_t *pgdat; 1158cf23422bSminskey guo int ret; 1159cf23422bSminskey guo 116001b0f197SToshi Kani if (node_online(nid)) 116101b0f197SToshi Kani return 0; 116201b0f197SToshi Kani 1163bfc8c901SVladimir Davydov mem_hotplug_begin(); 1164cf23422bSminskey guo pgdat = hotadd_new_pgdat(nid, 0); 11657553e8f2SDavid Rientjes if (!pgdat) { 116601b0f197SToshi Kani pr_err("Cannot online node %d due to NULL pgdat\n", nid); 1167cf23422bSminskey guo ret = -ENOMEM; 1168cf23422bSminskey guo goto out; 1169cf23422bSminskey guo } 1170cf23422bSminskey guo node_set_online(nid); 1171cf23422bSminskey guo ret = register_one_node(nid); 1172cf23422bSminskey guo BUG_ON(ret); 1173cf23422bSminskey guo 117401b0f197SToshi Kani if (pgdat->node_zonelists->_zonerefs->zone == NULL) { 117501b0f197SToshi Kani mutex_lock(&zonelists_mutex); 117601b0f197SToshi Kani build_all_zonelists(NULL, NULL); 117701b0f197SToshi Kani mutex_unlock(&zonelists_mutex); 117801b0f197SToshi Kani } 117901b0f197SToshi Kani 1180cf23422bSminskey guo out: 1181bfc8c901SVladimir Davydov mem_hotplug_done(); 1182cf23422bSminskey guo return ret; 1183cf23422bSminskey guo } 1184cf23422bSminskey guo 118527356f54SToshi Kani static int check_hotplug_memory_range(u64 start, u64 size) 118627356f54SToshi Kani { 1187c8e861a5SFabian Frederick u64 start_pfn = PFN_DOWN(start); 118827356f54SToshi Kani u64 nr_pages = size >> PAGE_SHIFT; 118927356f54SToshi Kani 119027356f54SToshi Kani /* Memory range must be aligned with section */ 119127356f54SToshi Kani if ((start_pfn & ~PAGE_SECTION_MASK) || 119227356f54SToshi Kani (nr_pages % PAGES_PER_SECTION) || (!nr_pages)) { 119327356f54SToshi Kani pr_err("Section-unaligned hotplug range: start 0x%llx, size 0x%llx\n", 119427356f54SToshi Kani (unsigned long long)start, 119527356f54SToshi Kani (unsigned long long)size); 119627356f54SToshi Kani return -EINVAL; 119727356f54SToshi Kani } 119827356f54SToshi Kani 119927356f54SToshi Kani return 0; 120027356f54SToshi Kani } 120127356f54SToshi Kani 120263264400SWang Nan /* 120363264400SWang Nan * If movable zone has already been setup, newly added memory should be check. 120463264400SWang Nan * If its address is higher than movable zone, it should be added as movable. 120563264400SWang Nan * Without this check, movable zone may overlap with other zone. 120663264400SWang Nan */ 120763264400SWang Nan static int should_add_memory_movable(int nid, u64 start, u64 size) 120863264400SWang Nan { 120963264400SWang Nan unsigned long start_pfn = start >> PAGE_SHIFT; 121063264400SWang Nan pg_data_t *pgdat = NODE_DATA(nid); 121163264400SWang Nan struct zone *movable_zone = pgdat->node_zones + ZONE_MOVABLE; 121263264400SWang Nan 121363264400SWang Nan if (zone_is_empty(movable_zone)) 121463264400SWang Nan return 0; 121563264400SWang Nan 121663264400SWang Nan if (movable_zone->zone_start_pfn <= start_pfn) 121763264400SWang Nan return 1; 121863264400SWang Nan 121963264400SWang Nan return 0; 122063264400SWang Nan } 122163264400SWang Nan 1222033fbae9SDan Williams int zone_for_memory(int nid, u64 start, u64 size, int zone_default, 1223033fbae9SDan Williams bool for_device) 122463264400SWang Nan { 1225033fbae9SDan Williams #ifdef CONFIG_ZONE_DEVICE 1226033fbae9SDan Williams if (for_device) 1227033fbae9SDan Williams return ZONE_DEVICE; 1228033fbae9SDan Williams #endif 122963264400SWang Nan if (should_add_memory_movable(nid, start, size)) 123063264400SWang Nan return ZONE_MOVABLE; 123163264400SWang Nan 123263264400SWang Nan return zone_default; 123363264400SWang Nan } 123463264400SWang Nan 123531168481SAl Viro /* we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG */ 123662cedb9fSDavid Vrabel int __ref add_memory_resource(int nid, struct resource *res) 1237bc02af93SYasunori Goto { 123862cedb9fSDavid Vrabel u64 start, size; 12399af3c2deSYasunori Goto pg_data_t *pgdat = NULL; 1240a1e565aaSTang Chen bool new_pgdat; 1241a1e565aaSTang Chen bool new_node; 1242bc02af93SYasunori Goto int ret; 1243bc02af93SYasunori Goto 124462cedb9fSDavid Vrabel start = res->start; 124562cedb9fSDavid Vrabel size = resource_size(res); 124662cedb9fSDavid Vrabel 124727356f54SToshi Kani ret = check_hotplug_memory_range(start, size); 124827356f54SToshi Kani if (ret) 124927356f54SToshi Kani return ret; 125027356f54SToshi Kani 1251a1e565aaSTang Chen { /* Stupid hack to suppress address-never-null warning */ 1252a1e565aaSTang Chen void *p = NODE_DATA(nid); 1253a1e565aaSTang Chen new_pgdat = !p; 1254a1e565aaSTang Chen } 1255ac13c462SNathan Zimmer 1256bfc8c901SVladimir Davydov mem_hotplug_begin(); 1257ac13c462SNathan Zimmer 12587f36e3e5STang Chen /* 12597f36e3e5STang Chen * Add new range to memblock so that when hotadd_new_pgdat() is called 12607f36e3e5STang Chen * to allocate new pgdat, get_pfn_range_for_nid() will be able to find 12617f36e3e5STang Chen * this new range and calculate total pages correctly. The range will 12627f36e3e5STang Chen * be removed at hot-remove time. 12637f36e3e5STang Chen */ 12647f36e3e5STang Chen memblock_add_node(start, size, nid); 12657f36e3e5STang Chen 1266a1e565aaSTang Chen new_node = !node_online(nid); 1267a1e565aaSTang Chen if (new_node) { 12689af3c2deSYasunori Goto pgdat = hotadd_new_pgdat(nid, start); 12696ad696d2SAndi Kleen ret = -ENOMEM; 12709af3c2deSYasunori Goto if (!pgdat) 127141b9e2d7SWen Congyang goto error; 12729af3c2deSYasunori Goto } 12739af3c2deSYasunori Goto 1274bc02af93SYasunori Goto /* call arch's memory hotadd */ 1275033fbae9SDan Williams ret = arch_add_memory(nid, start, size, false); 1276bc02af93SYasunori Goto 12779af3c2deSYasunori Goto if (ret < 0) 12789af3c2deSYasunori Goto goto error; 12799af3c2deSYasunori Goto 12800fc44159SYasunori Goto /* we online node here. we can't roll back from here. */ 12819af3c2deSYasunori Goto node_set_online(nid); 12829af3c2deSYasunori Goto 1283a1e565aaSTang Chen if (new_node) { 12840fc44159SYasunori Goto ret = register_one_node(nid); 12850fc44159SYasunori Goto /* 12860fc44159SYasunori Goto * If sysfs file of new node can't create, cpu on the node 12870fc44159SYasunori Goto * can't be hot-added. There is no rollback way now. 12880fc44159SYasunori Goto * So, check by BUG_ON() to catch it reluctantly.. 12890fc44159SYasunori Goto */ 12900fc44159SYasunori Goto BUG_ON(ret); 12910fc44159SYasunori Goto } 12920fc44159SYasunori Goto 1293d96ae530Sakpm@linux-foundation.org /* create new memmap entry */ 1294d96ae530Sakpm@linux-foundation.org firmware_map_add_hotplug(start, start + size, "System RAM"); 1295d96ae530Sakpm@linux-foundation.org 12966ad696d2SAndi Kleen goto out; 12976ad696d2SAndi Kleen 12989af3c2deSYasunori Goto error: 12999af3c2deSYasunori Goto /* rollback pgdat allocation and others */ 13009af3c2deSYasunori Goto if (new_pgdat) 13019af3c2deSYasunori Goto rollback_node_hotadd(nid, pgdat); 13027f36e3e5STang Chen memblock_remove(start, size); 13039af3c2deSYasunori Goto 13046ad696d2SAndi Kleen out: 1305bfc8c901SVladimir Davydov mem_hotplug_done(); 1306bc02af93SYasunori Goto return ret; 1307bc02af93SYasunori Goto } 130862cedb9fSDavid Vrabel EXPORT_SYMBOL_GPL(add_memory_resource); 130962cedb9fSDavid Vrabel 131062cedb9fSDavid Vrabel int __ref add_memory(int nid, u64 start, u64 size) 131162cedb9fSDavid Vrabel { 131262cedb9fSDavid Vrabel struct resource *res; 131362cedb9fSDavid Vrabel int ret; 131462cedb9fSDavid Vrabel 131562cedb9fSDavid Vrabel res = register_memory_resource(start, size); 1316*6f754ba4SVitaly Kuznetsov if (IS_ERR(res)) 1317*6f754ba4SVitaly Kuznetsov return PTR_ERR(res); 131862cedb9fSDavid Vrabel 131962cedb9fSDavid Vrabel ret = add_memory_resource(nid, res); 132062cedb9fSDavid Vrabel if (ret < 0) 132162cedb9fSDavid Vrabel release_memory_resource(res); 132262cedb9fSDavid Vrabel return ret; 132362cedb9fSDavid Vrabel } 1324bc02af93SYasunori Goto EXPORT_SYMBOL_GPL(add_memory); 13250c0e6195SKAMEZAWA Hiroyuki 13260c0e6195SKAMEZAWA Hiroyuki #ifdef CONFIG_MEMORY_HOTREMOVE 13270c0e6195SKAMEZAWA Hiroyuki /* 13285c755e9fSBadari Pulavarty * A free page on the buddy free lists (not the per-cpu lists) has PageBuddy 13295c755e9fSBadari Pulavarty * set and the size of the free page is given by page_order(). Using this, 13305c755e9fSBadari Pulavarty * the function determines if the pageblock contains only free pages. 13315c755e9fSBadari Pulavarty * Due to buddy contraints, a free page at least the size of a pageblock will 13325c755e9fSBadari Pulavarty * be located at the start of the pageblock 13335c755e9fSBadari Pulavarty */ 13345c755e9fSBadari Pulavarty static inline int pageblock_free(struct page *page) 13355c755e9fSBadari Pulavarty { 13365c755e9fSBadari Pulavarty return PageBuddy(page) && page_order(page) >= pageblock_order; 13375c755e9fSBadari Pulavarty } 13385c755e9fSBadari Pulavarty 13395c755e9fSBadari Pulavarty /* Return the start of the next active pageblock after a given page */ 13405c755e9fSBadari Pulavarty static struct page *next_active_pageblock(struct page *page) 13415c755e9fSBadari Pulavarty { 13425c755e9fSBadari Pulavarty /* Ensure the starting page is pageblock-aligned */ 13435c755e9fSBadari Pulavarty BUG_ON(page_to_pfn(page) & (pageblock_nr_pages - 1)); 13445c755e9fSBadari Pulavarty 13455c755e9fSBadari Pulavarty /* If the entire pageblock is free, move to the end of free page */ 13460dcc48c1SKAMEZAWA Hiroyuki if (pageblock_free(page)) { 13470dcc48c1SKAMEZAWA Hiroyuki int order; 13480dcc48c1SKAMEZAWA Hiroyuki /* be careful. we don't have locks, page_order can be changed.*/ 13490dcc48c1SKAMEZAWA Hiroyuki order = page_order(page); 13500dcc48c1SKAMEZAWA Hiroyuki if ((order < MAX_ORDER) && (order >= pageblock_order)) 13510dcc48c1SKAMEZAWA Hiroyuki return page + (1 << order); 13520dcc48c1SKAMEZAWA Hiroyuki } 13535c755e9fSBadari Pulavarty 13540dcc48c1SKAMEZAWA Hiroyuki return page + pageblock_nr_pages; 13555c755e9fSBadari Pulavarty } 13565c755e9fSBadari Pulavarty 13575c755e9fSBadari Pulavarty /* Checks if this range of memory is likely to be hot-removable. */ 13585c755e9fSBadari Pulavarty int is_mem_section_removable(unsigned long start_pfn, unsigned long nr_pages) 13595c755e9fSBadari Pulavarty { 13605c755e9fSBadari Pulavarty struct page *page = pfn_to_page(start_pfn); 13615c755e9fSBadari Pulavarty struct page *end_page = page + nr_pages; 13625c755e9fSBadari Pulavarty 13635c755e9fSBadari Pulavarty /* Check the starting page of each pageblock within the range */ 13645c755e9fSBadari Pulavarty for (; page < end_page; page = next_active_pageblock(page)) { 136549ac8255SKAMEZAWA Hiroyuki if (!is_pageblock_removable_nolock(page)) 13665c755e9fSBadari Pulavarty return 0; 136749ac8255SKAMEZAWA Hiroyuki cond_resched(); 13685c755e9fSBadari Pulavarty } 13695c755e9fSBadari Pulavarty 13705c755e9fSBadari Pulavarty /* All pageblocks in the memory block are likely to be hot-removable */ 13715c755e9fSBadari Pulavarty return 1; 13725c755e9fSBadari Pulavarty } 13735c755e9fSBadari Pulavarty 13745c755e9fSBadari Pulavarty /* 13750c0e6195SKAMEZAWA Hiroyuki * Confirm all pages in a range [start, end) is belongs to the same zone. 13760c0e6195SKAMEZAWA Hiroyuki */ 1377ed2f2400SZhang Zhen int test_pages_in_a_zone(unsigned long start_pfn, unsigned long end_pfn) 13780c0e6195SKAMEZAWA Hiroyuki { 13795f0f2887SAndrew Banman unsigned long pfn, sec_end_pfn; 13800c0e6195SKAMEZAWA Hiroyuki struct zone *zone = NULL; 13810c0e6195SKAMEZAWA Hiroyuki struct page *page; 13820c0e6195SKAMEZAWA Hiroyuki int i; 13835f0f2887SAndrew Banman for (pfn = start_pfn, sec_end_pfn = SECTION_ALIGN_UP(start_pfn); 13840c0e6195SKAMEZAWA Hiroyuki pfn < end_pfn; 13855f0f2887SAndrew Banman pfn = sec_end_pfn + 1, sec_end_pfn += PAGES_PER_SECTION) { 13865f0f2887SAndrew Banman /* Make sure the memory section is present first */ 13875f0f2887SAndrew Banman if (!present_section_nr(pfn_to_section_nr(pfn))) 13885f0f2887SAndrew Banman continue; 13895f0f2887SAndrew Banman for (; pfn < sec_end_pfn && pfn < end_pfn; 13900c0e6195SKAMEZAWA Hiroyuki pfn += MAX_ORDER_NR_PAGES) { 13910c0e6195SKAMEZAWA Hiroyuki i = 0; 13920c0e6195SKAMEZAWA Hiroyuki /* This is just a CONFIG_HOLES_IN_ZONE check.*/ 13935f0f2887SAndrew Banman while ((i < MAX_ORDER_NR_PAGES) && 13945f0f2887SAndrew Banman !pfn_valid_within(pfn + i)) 13950c0e6195SKAMEZAWA Hiroyuki i++; 13960c0e6195SKAMEZAWA Hiroyuki if (i == MAX_ORDER_NR_PAGES) 13970c0e6195SKAMEZAWA Hiroyuki continue; 13980c0e6195SKAMEZAWA Hiroyuki page = pfn_to_page(pfn + i); 13990c0e6195SKAMEZAWA Hiroyuki if (zone && page_zone(page) != zone) 14000c0e6195SKAMEZAWA Hiroyuki return 0; 14010c0e6195SKAMEZAWA Hiroyuki zone = page_zone(page); 14020c0e6195SKAMEZAWA Hiroyuki } 14035f0f2887SAndrew Banman } 14040c0e6195SKAMEZAWA Hiroyuki return 1; 14050c0e6195SKAMEZAWA Hiroyuki } 14060c0e6195SKAMEZAWA Hiroyuki 14070c0e6195SKAMEZAWA Hiroyuki /* 1408c8721bbbSNaoya Horiguchi * Scan pfn range [start,end) to find movable/migratable pages (LRU pages 1409c8721bbbSNaoya Horiguchi * and hugepages). We scan pfn because it's much easier than scanning over 1410c8721bbbSNaoya Horiguchi * linked list. This function returns the pfn of the first found movable 1411c8721bbbSNaoya Horiguchi * page if it's found, otherwise 0. 14120c0e6195SKAMEZAWA Hiroyuki */ 1413c8721bbbSNaoya Horiguchi static unsigned long scan_movable_pages(unsigned long start, unsigned long end) 14140c0e6195SKAMEZAWA Hiroyuki { 14150c0e6195SKAMEZAWA Hiroyuki unsigned long pfn; 14160c0e6195SKAMEZAWA Hiroyuki struct page *page; 14170c0e6195SKAMEZAWA Hiroyuki for (pfn = start; pfn < end; pfn++) { 14180c0e6195SKAMEZAWA Hiroyuki if (pfn_valid(pfn)) { 14190c0e6195SKAMEZAWA Hiroyuki page = pfn_to_page(pfn); 14200c0e6195SKAMEZAWA Hiroyuki if (PageLRU(page)) 14210c0e6195SKAMEZAWA Hiroyuki return pfn; 1422c8721bbbSNaoya Horiguchi if (PageHuge(page)) { 14237e1f049eSNaoya Horiguchi if (page_huge_active(page)) 1424c8721bbbSNaoya Horiguchi return pfn; 1425c8721bbbSNaoya Horiguchi else 1426c8721bbbSNaoya Horiguchi pfn = round_up(pfn + 1, 1427c8721bbbSNaoya Horiguchi 1 << compound_order(page)) - 1; 1428c8721bbbSNaoya Horiguchi } 14290c0e6195SKAMEZAWA Hiroyuki } 14300c0e6195SKAMEZAWA Hiroyuki } 14310c0e6195SKAMEZAWA Hiroyuki return 0; 14320c0e6195SKAMEZAWA Hiroyuki } 14330c0e6195SKAMEZAWA Hiroyuki 14340c0e6195SKAMEZAWA Hiroyuki #define NR_OFFLINE_AT_ONCE_PAGES (256) 14350c0e6195SKAMEZAWA Hiroyuki static int 14360c0e6195SKAMEZAWA Hiroyuki do_migrate_range(unsigned long start_pfn, unsigned long end_pfn) 14370c0e6195SKAMEZAWA Hiroyuki { 14380c0e6195SKAMEZAWA Hiroyuki unsigned long pfn; 14390c0e6195SKAMEZAWA Hiroyuki struct page *page; 14400c0e6195SKAMEZAWA Hiroyuki int move_pages = NR_OFFLINE_AT_ONCE_PAGES; 14410c0e6195SKAMEZAWA Hiroyuki int not_managed = 0; 14420c0e6195SKAMEZAWA Hiroyuki int ret = 0; 14430c0e6195SKAMEZAWA Hiroyuki LIST_HEAD(source); 14440c0e6195SKAMEZAWA Hiroyuki 14450c0e6195SKAMEZAWA Hiroyuki for (pfn = start_pfn; pfn < end_pfn && move_pages > 0; pfn++) { 14460c0e6195SKAMEZAWA Hiroyuki if (!pfn_valid(pfn)) 14470c0e6195SKAMEZAWA Hiroyuki continue; 14480c0e6195SKAMEZAWA Hiroyuki page = pfn_to_page(pfn); 1449c8721bbbSNaoya Horiguchi 1450c8721bbbSNaoya Horiguchi if (PageHuge(page)) { 1451c8721bbbSNaoya Horiguchi struct page *head = compound_head(page); 1452c8721bbbSNaoya Horiguchi pfn = page_to_pfn(head) + (1<<compound_order(head)) - 1; 1453c8721bbbSNaoya Horiguchi if (compound_order(head) > PFN_SECTION_SHIFT) { 1454c8721bbbSNaoya Horiguchi ret = -EBUSY; 1455c8721bbbSNaoya Horiguchi break; 1456c8721bbbSNaoya Horiguchi } 1457c8721bbbSNaoya Horiguchi if (isolate_huge_page(page, &source)) 1458c8721bbbSNaoya Horiguchi move_pages -= 1 << compound_order(head); 1459c8721bbbSNaoya Horiguchi continue; 1460c8721bbbSNaoya Horiguchi } 1461c8721bbbSNaoya Horiguchi 1462700c2a46SKonstantin Khlebnikov if (!get_page_unless_zero(page)) 14630c0e6195SKAMEZAWA Hiroyuki continue; 14640c0e6195SKAMEZAWA Hiroyuki /* 14650c0e6195SKAMEZAWA Hiroyuki * We can skip free pages. And we can only deal with pages on 14660c0e6195SKAMEZAWA Hiroyuki * LRU. 14670c0e6195SKAMEZAWA Hiroyuki */ 146862695a84SNick Piggin ret = isolate_lru_page(page); 14690c0e6195SKAMEZAWA Hiroyuki if (!ret) { /* Success */ 1470700c2a46SKonstantin Khlebnikov put_page(page); 147162695a84SNick Piggin list_add_tail(&page->lru, &source); 14720c0e6195SKAMEZAWA Hiroyuki move_pages--; 14736d9c285aSKOSAKI Motohiro inc_zone_page_state(page, NR_ISOLATED_ANON + 14746d9c285aSKOSAKI Motohiro page_is_file_cache(page)); 14756d9c285aSKOSAKI Motohiro 14760c0e6195SKAMEZAWA Hiroyuki } else { 14770c0e6195SKAMEZAWA Hiroyuki #ifdef CONFIG_DEBUG_VM 1478718a3821SWu Fengguang printk(KERN_ALERT "removing pfn %lx from LRU failed\n", 1479718a3821SWu Fengguang pfn); 1480f0b791a3SDave Hansen dump_page(page, "failed to remove from LRU"); 14810c0e6195SKAMEZAWA Hiroyuki #endif 1482700c2a46SKonstantin Khlebnikov put_page(page); 148325985edcSLucas De Marchi /* Because we don't have big zone->lock. we should 1484809c4449SBob Liu check this again here. */ 1485809c4449SBob Liu if (page_count(page)) { 1486809c4449SBob Liu not_managed++; 1487f3ab2636SBob Liu ret = -EBUSY; 1488809c4449SBob Liu break; 1489809c4449SBob Liu } 14900c0e6195SKAMEZAWA Hiroyuki } 14910c0e6195SKAMEZAWA Hiroyuki } 1492f3ab2636SBob Liu if (!list_empty(&source)) { 14930c0e6195SKAMEZAWA Hiroyuki if (not_managed) { 1494c8721bbbSNaoya Horiguchi putback_movable_pages(&source); 14950c0e6195SKAMEZAWA Hiroyuki goto out; 14960c0e6195SKAMEZAWA Hiroyuki } 149774c08f98SMinchan Kim 149874c08f98SMinchan Kim /* 149974c08f98SMinchan Kim * alloc_migrate_target should be improooooved!! 150074c08f98SMinchan Kim * migrate_pages returns # of failed pages. 150174c08f98SMinchan Kim */ 150268711a74SDavid Rientjes ret = migrate_pages(&source, alloc_migrate_target, NULL, 0, 15039c620e2bSHugh Dickins MIGRATE_SYNC, MR_MEMORY_HOTPLUG); 1504cf608ac1SMinchan Kim if (ret) 1505c8721bbbSNaoya Horiguchi putback_movable_pages(&source); 1506f3ab2636SBob Liu } 15070c0e6195SKAMEZAWA Hiroyuki out: 15080c0e6195SKAMEZAWA Hiroyuki return ret; 15090c0e6195SKAMEZAWA Hiroyuki } 15100c0e6195SKAMEZAWA Hiroyuki 15110c0e6195SKAMEZAWA Hiroyuki /* 15120c0e6195SKAMEZAWA Hiroyuki * remove from free_area[] and mark all as Reserved. 15130c0e6195SKAMEZAWA Hiroyuki */ 15140c0e6195SKAMEZAWA Hiroyuki static int 15150c0e6195SKAMEZAWA Hiroyuki offline_isolated_pages_cb(unsigned long start, unsigned long nr_pages, 15160c0e6195SKAMEZAWA Hiroyuki void *data) 15170c0e6195SKAMEZAWA Hiroyuki { 15180c0e6195SKAMEZAWA Hiroyuki __offline_isolated_pages(start, start + nr_pages); 15190c0e6195SKAMEZAWA Hiroyuki return 0; 15200c0e6195SKAMEZAWA Hiroyuki } 15210c0e6195SKAMEZAWA Hiroyuki 15220c0e6195SKAMEZAWA Hiroyuki static void 15230c0e6195SKAMEZAWA Hiroyuki offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn) 15240c0e6195SKAMEZAWA Hiroyuki { 1525908eedc6SKAMEZAWA Hiroyuki walk_system_ram_range(start_pfn, end_pfn - start_pfn, NULL, 15260c0e6195SKAMEZAWA Hiroyuki offline_isolated_pages_cb); 15270c0e6195SKAMEZAWA Hiroyuki } 15280c0e6195SKAMEZAWA Hiroyuki 15290c0e6195SKAMEZAWA Hiroyuki /* 15300c0e6195SKAMEZAWA Hiroyuki * Check all pages in range, recoreded as memory resource, are isolated. 15310c0e6195SKAMEZAWA Hiroyuki */ 15320c0e6195SKAMEZAWA Hiroyuki static int 15330c0e6195SKAMEZAWA Hiroyuki check_pages_isolated_cb(unsigned long start_pfn, unsigned long nr_pages, 15340c0e6195SKAMEZAWA Hiroyuki void *data) 15350c0e6195SKAMEZAWA Hiroyuki { 15360c0e6195SKAMEZAWA Hiroyuki int ret; 15370c0e6195SKAMEZAWA Hiroyuki long offlined = *(long *)data; 1538b023f468SWen Congyang ret = test_pages_isolated(start_pfn, start_pfn + nr_pages, true); 15390c0e6195SKAMEZAWA Hiroyuki offlined = nr_pages; 15400c0e6195SKAMEZAWA Hiroyuki if (!ret) 15410c0e6195SKAMEZAWA Hiroyuki *(long *)data += offlined; 15420c0e6195SKAMEZAWA Hiroyuki return ret; 15430c0e6195SKAMEZAWA Hiroyuki } 15440c0e6195SKAMEZAWA Hiroyuki 15450c0e6195SKAMEZAWA Hiroyuki static long 15460c0e6195SKAMEZAWA Hiroyuki check_pages_isolated(unsigned long start_pfn, unsigned long end_pfn) 15470c0e6195SKAMEZAWA Hiroyuki { 15480c0e6195SKAMEZAWA Hiroyuki long offlined = 0; 15490c0e6195SKAMEZAWA Hiroyuki int ret; 15500c0e6195SKAMEZAWA Hiroyuki 1551908eedc6SKAMEZAWA Hiroyuki ret = walk_system_ram_range(start_pfn, end_pfn - start_pfn, &offlined, 15520c0e6195SKAMEZAWA Hiroyuki check_pages_isolated_cb); 15530c0e6195SKAMEZAWA Hiroyuki if (ret < 0) 15540c0e6195SKAMEZAWA Hiroyuki offlined = (long)ret; 15550c0e6195SKAMEZAWA Hiroyuki return offlined; 15560c0e6195SKAMEZAWA Hiroyuki } 15570c0e6195SKAMEZAWA Hiroyuki 155809285af7SLai Jiangshan #ifdef CONFIG_MOVABLE_NODE 155979a4dcefSTang Chen /* 156079a4dcefSTang Chen * When CONFIG_MOVABLE_NODE, we permit offlining of a node which doesn't have 156179a4dcefSTang Chen * normal memory. 156279a4dcefSTang Chen */ 156309285af7SLai Jiangshan static bool can_offline_normal(struct zone *zone, unsigned long nr_pages) 156409285af7SLai Jiangshan { 156509285af7SLai Jiangshan return true; 156609285af7SLai Jiangshan } 156779a4dcefSTang Chen #else /* CONFIG_MOVABLE_NODE */ 156874d42d8fSLai Jiangshan /* ensure the node has NORMAL memory if it is still online */ 156974d42d8fSLai Jiangshan static bool can_offline_normal(struct zone *zone, unsigned long nr_pages) 157074d42d8fSLai Jiangshan { 157174d42d8fSLai Jiangshan struct pglist_data *pgdat = zone->zone_pgdat; 157274d42d8fSLai Jiangshan unsigned long present_pages = 0; 157374d42d8fSLai Jiangshan enum zone_type zt; 157474d42d8fSLai Jiangshan 157574d42d8fSLai Jiangshan for (zt = 0; zt <= ZONE_NORMAL; zt++) 157674d42d8fSLai Jiangshan present_pages += pgdat->node_zones[zt].present_pages; 157774d42d8fSLai Jiangshan 157874d42d8fSLai Jiangshan if (present_pages > nr_pages) 157974d42d8fSLai Jiangshan return true; 158074d42d8fSLai Jiangshan 158174d42d8fSLai Jiangshan present_pages = 0; 158274d42d8fSLai Jiangshan for (; zt <= ZONE_MOVABLE; zt++) 158374d42d8fSLai Jiangshan present_pages += pgdat->node_zones[zt].present_pages; 158474d42d8fSLai Jiangshan 158574d42d8fSLai Jiangshan /* 158674d42d8fSLai Jiangshan * we can't offline the last normal memory until all 158774d42d8fSLai Jiangshan * higher memory is offlined. 158874d42d8fSLai Jiangshan */ 158974d42d8fSLai Jiangshan return present_pages == 0; 159074d42d8fSLai Jiangshan } 159179a4dcefSTang Chen #endif /* CONFIG_MOVABLE_NODE */ 159274d42d8fSLai Jiangshan 1593c5320926STang Chen static int __init cmdline_parse_movable_node(char *p) 1594c5320926STang Chen { 1595c5320926STang Chen #ifdef CONFIG_MOVABLE_NODE 1596c5320926STang Chen /* 1597c5320926STang Chen * Memory used by the kernel cannot be hot-removed because Linux 1598c5320926STang Chen * cannot migrate the kernel pages. When memory hotplug is 1599c5320926STang Chen * enabled, we should prevent memblock from allocating memory 1600c5320926STang Chen * for the kernel. 1601c5320926STang Chen * 1602c5320926STang Chen * ACPI SRAT records all hotpluggable memory ranges. But before 1603c5320926STang Chen * SRAT is parsed, we don't know about it. 1604c5320926STang Chen * 1605c5320926STang Chen * The kernel image is loaded into memory at very early time. We 1606c5320926STang Chen * cannot prevent this anyway. So on NUMA system, we set any 1607c5320926STang Chen * node the kernel resides in as un-hotpluggable. 1608c5320926STang Chen * 1609c5320926STang Chen * Since on modern servers, one node could have double-digit 1610c5320926STang Chen * gigabytes memory, we can assume the memory around the kernel 1611c5320926STang Chen * image is also un-hotpluggable. So before SRAT is parsed, just 1612c5320926STang Chen * allocate memory near the kernel image to try the best to keep 1613c5320926STang Chen * the kernel away from hotpluggable memory. 1614c5320926STang Chen */ 1615c5320926STang Chen memblock_set_bottom_up(true); 161655ac590cSTang Chen movable_node_enabled = true; 1617c5320926STang Chen #else 1618c5320926STang Chen pr_warn("movable_node option not supported\n"); 1619c5320926STang Chen #endif 1620c5320926STang Chen return 0; 1621c5320926STang Chen } 1622c5320926STang Chen early_param("movable_node", cmdline_parse_movable_node); 1623c5320926STang Chen 1624d9713679SLai Jiangshan /* check which state of node_states will be changed when offline memory */ 1625d9713679SLai Jiangshan static void node_states_check_changes_offline(unsigned long nr_pages, 1626d9713679SLai Jiangshan struct zone *zone, struct memory_notify *arg) 1627d9713679SLai Jiangshan { 1628d9713679SLai Jiangshan struct pglist_data *pgdat = zone->zone_pgdat; 1629d9713679SLai Jiangshan unsigned long present_pages = 0; 1630d9713679SLai Jiangshan enum zone_type zt, zone_last = ZONE_NORMAL; 1631d9713679SLai Jiangshan 1632d9713679SLai Jiangshan /* 16336715ddf9SLai Jiangshan * If we have HIGHMEM or movable node, node_states[N_NORMAL_MEMORY] 16346715ddf9SLai Jiangshan * contains nodes which have zones of 0...ZONE_NORMAL, 16356715ddf9SLai Jiangshan * set zone_last to ZONE_NORMAL. 1636d9713679SLai Jiangshan * 16376715ddf9SLai Jiangshan * If we don't have HIGHMEM nor movable node, 16386715ddf9SLai Jiangshan * node_states[N_NORMAL_MEMORY] contains nodes which have zones of 16396715ddf9SLai Jiangshan * 0...ZONE_MOVABLE, set zone_last to ZONE_MOVABLE. 1640d9713679SLai Jiangshan */ 16416715ddf9SLai Jiangshan if (N_MEMORY == N_NORMAL_MEMORY) 1642d9713679SLai Jiangshan zone_last = ZONE_MOVABLE; 1643d9713679SLai Jiangshan 1644d9713679SLai Jiangshan /* 1645d9713679SLai Jiangshan * check whether node_states[N_NORMAL_MEMORY] will be changed. 1646d9713679SLai Jiangshan * If the memory to be offline is in a zone of 0...zone_last, 1647d9713679SLai Jiangshan * and it is the last present memory, 0...zone_last will 1648d9713679SLai Jiangshan * become empty after offline , thus we can determind we will 1649d9713679SLai Jiangshan * need to clear the node from node_states[N_NORMAL_MEMORY]. 1650d9713679SLai Jiangshan */ 1651d9713679SLai Jiangshan for (zt = 0; zt <= zone_last; zt++) 1652d9713679SLai Jiangshan present_pages += pgdat->node_zones[zt].present_pages; 1653d9713679SLai Jiangshan if (zone_idx(zone) <= zone_last && nr_pages >= present_pages) 1654d9713679SLai Jiangshan arg->status_change_nid_normal = zone_to_nid(zone); 1655d9713679SLai Jiangshan else 1656d9713679SLai Jiangshan arg->status_change_nid_normal = -1; 1657d9713679SLai Jiangshan 16586715ddf9SLai Jiangshan #ifdef CONFIG_HIGHMEM 16596715ddf9SLai Jiangshan /* 16606715ddf9SLai Jiangshan * If we have movable node, node_states[N_HIGH_MEMORY] 16616715ddf9SLai Jiangshan * contains nodes which have zones of 0...ZONE_HIGHMEM, 16626715ddf9SLai Jiangshan * set zone_last to ZONE_HIGHMEM. 16636715ddf9SLai Jiangshan * 16646715ddf9SLai Jiangshan * If we don't have movable node, node_states[N_NORMAL_MEMORY] 16656715ddf9SLai Jiangshan * contains nodes which have zones of 0...ZONE_MOVABLE, 16666715ddf9SLai Jiangshan * set zone_last to ZONE_MOVABLE. 16676715ddf9SLai Jiangshan */ 16686715ddf9SLai Jiangshan zone_last = ZONE_HIGHMEM; 16696715ddf9SLai Jiangshan if (N_MEMORY == N_HIGH_MEMORY) 16706715ddf9SLai Jiangshan zone_last = ZONE_MOVABLE; 16716715ddf9SLai Jiangshan 16726715ddf9SLai Jiangshan for (; zt <= zone_last; zt++) 16736715ddf9SLai Jiangshan present_pages += pgdat->node_zones[zt].present_pages; 16746715ddf9SLai Jiangshan if (zone_idx(zone) <= zone_last && nr_pages >= present_pages) 16756715ddf9SLai Jiangshan arg->status_change_nid_high = zone_to_nid(zone); 16766715ddf9SLai Jiangshan else 16776715ddf9SLai Jiangshan arg->status_change_nid_high = -1; 16786715ddf9SLai Jiangshan #else 16796715ddf9SLai Jiangshan arg->status_change_nid_high = arg->status_change_nid_normal; 16806715ddf9SLai Jiangshan #endif 16816715ddf9SLai Jiangshan 1682d9713679SLai Jiangshan /* 1683d9713679SLai Jiangshan * node_states[N_HIGH_MEMORY] contains nodes which have 0...ZONE_MOVABLE 1684d9713679SLai Jiangshan */ 1685d9713679SLai Jiangshan zone_last = ZONE_MOVABLE; 1686d9713679SLai Jiangshan 1687d9713679SLai Jiangshan /* 1688d9713679SLai Jiangshan * check whether node_states[N_HIGH_MEMORY] will be changed 1689d9713679SLai Jiangshan * If we try to offline the last present @nr_pages from the node, 1690d9713679SLai Jiangshan * we can determind we will need to clear the node from 1691d9713679SLai Jiangshan * node_states[N_HIGH_MEMORY]. 1692d9713679SLai Jiangshan */ 1693d9713679SLai Jiangshan for (; zt <= zone_last; zt++) 1694d9713679SLai Jiangshan present_pages += pgdat->node_zones[zt].present_pages; 1695d9713679SLai Jiangshan if (nr_pages >= present_pages) 1696d9713679SLai Jiangshan arg->status_change_nid = zone_to_nid(zone); 1697d9713679SLai Jiangshan else 1698d9713679SLai Jiangshan arg->status_change_nid = -1; 1699d9713679SLai Jiangshan } 1700d9713679SLai Jiangshan 1701d9713679SLai Jiangshan static void node_states_clear_node(int node, struct memory_notify *arg) 1702d9713679SLai Jiangshan { 1703d9713679SLai Jiangshan if (arg->status_change_nid_normal >= 0) 1704d9713679SLai Jiangshan node_clear_state(node, N_NORMAL_MEMORY); 1705d9713679SLai Jiangshan 17066715ddf9SLai Jiangshan if ((N_MEMORY != N_NORMAL_MEMORY) && 17076715ddf9SLai Jiangshan (arg->status_change_nid_high >= 0)) 1708d9713679SLai Jiangshan node_clear_state(node, N_HIGH_MEMORY); 17096715ddf9SLai Jiangshan 17106715ddf9SLai Jiangshan if ((N_MEMORY != N_HIGH_MEMORY) && 17116715ddf9SLai Jiangshan (arg->status_change_nid >= 0)) 17126715ddf9SLai Jiangshan node_clear_state(node, N_MEMORY); 1713d9713679SLai Jiangshan } 1714d9713679SLai Jiangshan 1715a16cee10SWen Congyang static int __ref __offline_pages(unsigned long start_pfn, 17160c0e6195SKAMEZAWA Hiroyuki unsigned long end_pfn, unsigned long timeout) 17170c0e6195SKAMEZAWA Hiroyuki { 17180c0e6195SKAMEZAWA Hiroyuki unsigned long pfn, nr_pages, expire; 17190c0e6195SKAMEZAWA Hiroyuki long offlined_pages; 17207b78d335SYasunori Goto int ret, drain, retry_max, node; 1721d702909fSCody P Schafer unsigned long flags; 17220c0e6195SKAMEZAWA Hiroyuki struct zone *zone; 17237b78d335SYasunori Goto struct memory_notify arg; 17240c0e6195SKAMEZAWA Hiroyuki 17250c0e6195SKAMEZAWA Hiroyuki /* at least, alignment against pageblock is necessary */ 17260c0e6195SKAMEZAWA Hiroyuki if (!IS_ALIGNED(start_pfn, pageblock_nr_pages)) 17270c0e6195SKAMEZAWA Hiroyuki return -EINVAL; 17280c0e6195SKAMEZAWA Hiroyuki if (!IS_ALIGNED(end_pfn, pageblock_nr_pages)) 17290c0e6195SKAMEZAWA Hiroyuki return -EINVAL; 17300c0e6195SKAMEZAWA Hiroyuki /* This makes hotplug much easier...and readable. 17310c0e6195SKAMEZAWA Hiroyuki we assume this for now. .*/ 17320c0e6195SKAMEZAWA Hiroyuki if (!test_pages_in_a_zone(start_pfn, end_pfn)) 17330c0e6195SKAMEZAWA Hiroyuki return -EINVAL; 17347b78d335SYasunori Goto 17357b78d335SYasunori Goto zone = page_zone(pfn_to_page(start_pfn)); 17367b78d335SYasunori Goto node = zone_to_nid(zone); 17377b78d335SYasunori Goto nr_pages = end_pfn - start_pfn; 17387b78d335SYasunori Goto 173974d42d8fSLai Jiangshan if (zone_idx(zone) <= ZONE_NORMAL && !can_offline_normal(zone, nr_pages)) 174030467e0bSDavid Rientjes return -EINVAL; 174174d42d8fSLai Jiangshan 17420c0e6195SKAMEZAWA Hiroyuki /* set above range as isolated */ 1743b023f468SWen Congyang ret = start_isolate_page_range(start_pfn, end_pfn, 1744b023f468SWen Congyang MIGRATE_MOVABLE, true); 17450c0e6195SKAMEZAWA Hiroyuki if (ret) 174630467e0bSDavid Rientjes return ret; 17477b78d335SYasunori Goto 17487b78d335SYasunori Goto arg.start_pfn = start_pfn; 17497b78d335SYasunori Goto arg.nr_pages = nr_pages; 1750d9713679SLai Jiangshan node_states_check_changes_offline(nr_pages, zone, &arg); 17517b78d335SYasunori Goto 17527b78d335SYasunori Goto ret = memory_notify(MEM_GOING_OFFLINE, &arg); 17537b78d335SYasunori Goto ret = notifier_to_errno(ret); 17547b78d335SYasunori Goto if (ret) 17557b78d335SYasunori Goto goto failed_removal; 17567b78d335SYasunori Goto 17570c0e6195SKAMEZAWA Hiroyuki pfn = start_pfn; 17580c0e6195SKAMEZAWA Hiroyuki expire = jiffies + timeout; 17590c0e6195SKAMEZAWA Hiroyuki drain = 0; 17600c0e6195SKAMEZAWA Hiroyuki retry_max = 5; 17610c0e6195SKAMEZAWA Hiroyuki repeat: 17620c0e6195SKAMEZAWA Hiroyuki /* start memory hot removal */ 17630c0e6195SKAMEZAWA Hiroyuki ret = -EAGAIN; 17640c0e6195SKAMEZAWA Hiroyuki if (time_after(jiffies, expire)) 17650c0e6195SKAMEZAWA Hiroyuki goto failed_removal; 17660c0e6195SKAMEZAWA Hiroyuki ret = -EINTR; 17670c0e6195SKAMEZAWA Hiroyuki if (signal_pending(current)) 17680c0e6195SKAMEZAWA Hiroyuki goto failed_removal; 17690c0e6195SKAMEZAWA Hiroyuki ret = 0; 17700c0e6195SKAMEZAWA Hiroyuki if (drain) { 17710c0e6195SKAMEZAWA Hiroyuki lru_add_drain_all(); 17720c0e6195SKAMEZAWA Hiroyuki cond_resched(); 1773c0554329SVlastimil Babka drain_all_pages(zone); 17740c0e6195SKAMEZAWA Hiroyuki } 17750c0e6195SKAMEZAWA Hiroyuki 1776c8721bbbSNaoya Horiguchi pfn = scan_movable_pages(start_pfn, end_pfn); 1777c8721bbbSNaoya Horiguchi if (pfn) { /* We have movable pages */ 17780c0e6195SKAMEZAWA Hiroyuki ret = do_migrate_range(pfn, end_pfn); 17790c0e6195SKAMEZAWA Hiroyuki if (!ret) { 17800c0e6195SKAMEZAWA Hiroyuki drain = 1; 17810c0e6195SKAMEZAWA Hiroyuki goto repeat; 17820c0e6195SKAMEZAWA Hiroyuki } else { 17830c0e6195SKAMEZAWA Hiroyuki if (ret < 0) 17840c0e6195SKAMEZAWA Hiroyuki if (--retry_max == 0) 17850c0e6195SKAMEZAWA Hiroyuki goto failed_removal; 17860c0e6195SKAMEZAWA Hiroyuki yield(); 17870c0e6195SKAMEZAWA Hiroyuki drain = 1; 17880c0e6195SKAMEZAWA Hiroyuki goto repeat; 17890c0e6195SKAMEZAWA Hiroyuki } 17900c0e6195SKAMEZAWA Hiroyuki } 1791b3834be5SAdam Buchbinder /* drain all zone's lru pagevec, this is asynchronous... */ 17920c0e6195SKAMEZAWA Hiroyuki lru_add_drain_all(); 17930c0e6195SKAMEZAWA Hiroyuki yield(); 1794b3834be5SAdam Buchbinder /* drain pcp pages, this is synchronous. */ 1795c0554329SVlastimil Babka drain_all_pages(zone); 1796c8721bbbSNaoya Horiguchi /* 1797c8721bbbSNaoya Horiguchi * dissolve free hugepages in the memory block before doing offlining 1798c8721bbbSNaoya Horiguchi * actually in order to make hugetlbfs's object counting consistent. 1799c8721bbbSNaoya Horiguchi */ 1800c8721bbbSNaoya Horiguchi dissolve_free_huge_pages(start_pfn, end_pfn); 18010c0e6195SKAMEZAWA Hiroyuki /* check again */ 18020c0e6195SKAMEZAWA Hiroyuki offlined_pages = check_pages_isolated(start_pfn, end_pfn); 18030c0e6195SKAMEZAWA Hiroyuki if (offlined_pages < 0) { 18040c0e6195SKAMEZAWA Hiroyuki ret = -EBUSY; 18050c0e6195SKAMEZAWA Hiroyuki goto failed_removal; 18060c0e6195SKAMEZAWA Hiroyuki } 18070c0e6195SKAMEZAWA Hiroyuki printk(KERN_INFO "Offlined Pages %ld\n", offlined_pages); 1808b3834be5SAdam Buchbinder /* Ok, all of our target is isolated. 18090c0e6195SKAMEZAWA Hiroyuki We cannot do rollback at this point. */ 18100c0e6195SKAMEZAWA Hiroyuki offline_isolated_pages(start_pfn, end_pfn); 1811dbc0e4ceSKAMEZAWA Hiroyuki /* reset pagetype flags and makes migrate type to be MOVABLE */ 18120815f3d8SMichal Nazarewicz undo_isolate_page_range(start_pfn, end_pfn, MIGRATE_MOVABLE); 18130c0e6195SKAMEZAWA Hiroyuki /* removal success */ 18143dcc0571SJiang Liu adjust_managed_page_count(pfn_to_page(start_pfn), -offlined_pages); 18150c0e6195SKAMEZAWA Hiroyuki zone->present_pages -= offlined_pages; 1816d702909fSCody P Schafer 1817d702909fSCody P Schafer pgdat_resize_lock(zone->zone_pgdat, &flags); 18180c0e6195SKAMEZAWA Hiroyuki zone->zone_pgdat->node_present_pages -= offlined_pages; 1819d702909fSCody P Schafer pgdat_resize_unlock(zone->zone_pgdat, &flags); 18207b78d335SYasunori Goto 18211b79acc9SKOSAKI Motohiro init_per_zone_wmark_min(); 18221b79acc9SKOSAKI Motohiro 18231e8537baSXishi Qiu if (!populated_zone(zone)) { 1824340175b7SJiang Liu zone_pcp_reset(zone); 18251e8537baSXishi Qiu mutex_lock(&zonelists_mutex); 18261e8537baSXishi Qiu build_all_zonelists(NULL, NULL); 18271e8537baSXishi Qiu mutex_unlock(&zonelists_mutex); 18281e8537baSXishi Qiu } else 18291e8537baSXishi Qiu zone_pcp_update(zone); 1830340175b7SJiang Liu 1831d9713679SLai Jiangshan node_states_clear_node(node, &arg); 1832d9713679SLai Jiangshan if (arg.status_change_nid >= 0) 18338fe23e05SDavid Rientjes kswapd_stop(node); 1834bce7394aSMinchan Kim 18350c0e6195SKAMEZAWA Hiroyuki vm_total_pages = nr_free_pagecache_pages(); 18360c0e6195SKAMEZAWA Hiroyuki writeback_set_ratelimit(); 18377b78d335SYasunori Goto 18387b78d335SYasunori Goto memory_notify(MEM_OFFLINE, &arg); 18390c0e6195SKAMEZAWA Hiroyuki return 0; 18400c0e6195SKAMEZAWA Hiroyuki 18410c0e6195SKAMEZAWA Hiroyuki failed_removal: 1842a62e2f4fSBjorn Helgaas printk(KERN_INFO "memory offlining [mem %#010llx-%#010llx] failed\n", 1843a62e2f4fSBjorn Helgaas (unsigned long long) start_pfn << PAGE_SHIFT, 1844a62e2f4fSBjorn Helgaas ((unsigned long long) end_pfn << PAGE_SHIFT) - 1); 18457b78d335SYasunori Goto memory_notify(MEM_CANCEL_OFFLINE, &arg); 18460c0e6195SKAMEZAWA Hiroyuki /* pushback to free area */ 18470815f3d8SMichal Nazarewicz undo_isolate_page_range(start_pfn, end_pfn, MIGRATE_MOVABLE); 18480c0e6195SKAMEZAWA Hiroyuki return ret; 18490c0e6195SKAMEZAWA Hiroyuki } 185071088785SBadari Pulavarty 185130467e0bSDavid Rientjes /* Must be protected by mem_hotplug_begin() */ 1852a16cee10SWen Congyang int offline_pages(unsigned long start_pfn, unsigned long nr_pages) 1853a16cee10SWen Congyang { 1854a16cee10SWen Congyang return __offline_pages(start_pfn, start_pfn + nr_pages, 120 * HZ); 1855a16cee10SWen Congyang } 1856e2ff3940SRafael J. Wysocki #endif /* CONFIG_MEMORY_HOTREMOVE */ 1857a16cee10SWen Congyang 1858bbc76be6SWen Congyang /** 1859bbc76be6SWen Congyang * walk_memory_range - walks through all mem sections in [start_pfn, end_pfn) 1860bbc76be6SWen Congyang * @start_pfn: start pfn of the memory range 1861e05c4bbfSToshi Kani * @end_pfn: end pfn of the memory range 1862bbc76be6SWen Congyang * @arg: argument passed to func 1863bbc76be6SWen Congyang * @func: callback for each memory section walked 1864bbc76be6SWen Congyang * 1865bbc76be6SWen Congyang * This function walks through all present mem sections in range 1866bbc76be6SWen Congyang * [start_pfn, end_pfn) and call func on each mem section. 1867bbc76be6SWen Congyang * 1868bbc76be6SWen Congyang * Returns the return value of func. 1869bbc76be6SWen Congyang */ 1870e2ff3940SRafael J. Wysocki int walk_memory_range(unsigned long start_pfn, unsigned long end_pfn, 1871bbc76be6SWen Congyang void *arg, int (*func)(struct memory_block *, void *)) 187271088785SBadari Pulavarty { 1873e90bdb7fSWen Congyang struct memory_block *mem = NULL; 1874e90bdb7fSWen Congyang struct mem_section *section; 1875e90bdb7fSWen Congyang unsigned long pfn, section_nr; 1876e90bdb7fSWen Congyang int ret; 187771088785SBadari Pulavarty 1878e90bdb7fSWen Congyang for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) { 1879e90bdb7fSWen Congyang section_nr = pfn_to_section_nr(pfn); 1880e90bdb7fSWen Congyang if (!present_section_nr(section_nr)) 1881e90bdb7fSWen Congyang continue; 1882e90bdb7fSWen Congyang 1883e90bdb7fSWen Congyang section = __nr_to_section(section_nr); 1884e90bdb7fSWen Congyang /* same memblock? */ 1885e90bdb7fSWen Congyang if (mem) 1886e90bdb7fSWen Congyang if ((section_nr >= mem->start_section_nr) && 1887e90bdb7fSWen Congyang (section_nr <= mem->end_section_nr)) 1888e90bdb7fSWen Congyang continue; 1889e90bdb7fSWen Congyang 1890e90bdb7fSWen Congyang mem = find_memory_block_hinted(section, mem); 1891e90bdb7fSWen Congyang if (!mem) 1892e90bdb7fSWen Congyang continue; 1893e90bdb7fSWen Congyang 1894bbc76be6SWen Congyang ret = func(mem, arg); 1895e90bdb7fSWen Congyang if (ret) { 1896e90bdb7fSWen Congyang kobject_put(&mem->dev.kobj); 1897e90bdb7fSWen Congyang return ret; 1898e90bdb7fSWen Congyang } 1899e90bdb7fSWen Congyang } 1900e90bdb7fSWen Congyang 1901e90bdb7fSWen Congyang if (mem) 1902e90bdb7fSWen Congyang kobject_put(&mem->dev.kobj); 1903e90bdb7fSWen Congyang 1904bbc76be6SWen Congyang return 0; 1905bbc76be6SWen Congyang } 1906bbc76be6SWen Congyang 1907e2ff3940SRafael J. Wysocki #ifdef CONFIG_MEMORY_HOTREMOVE 1908d6de9d53SXishi Qiu static int check_memblock_offlined_cb(struct memory_block *mem, void *arg) 1909bbc76be6SWen Congyang { 1910bbc76be6SWen Congyang int ret = !is_memblock_offlined(mem); 1911bbc76be6SWen Congyang 1912349daa0fSRandy Dunlap if (unlikely(ret)) { 1913349daa0fSRandy Dunlap phys_addr_t beginpa, endpa; 1914349daa0fSRandy Dunlap 1915349daa0fSRandy Dunlap beginpa = PFN_PHYS(section_nr_to_pfn(mem->start_section_nr)); 1916349daa0fSRandy Dunlap endpa = PFN_PHYS(section_nr_to_pfn(mem->end_section_nr + 1))-1; 1917bbc76be6SWen Congyang pr_warn("removing memory fails, because memory " 1918349daa0fSRandy Dunlap "[%pa-%pa] is onlined\n", 1919349daa0fSRandy Dunlap &beginpa, &endpa); 1920349daa0fSRandy Dunlap } 1921bbc76be6SWen Congyang 1922bbc76be6SWen Congyang return ret; 1923bbc76be6SWen Congyang } 1924bbc76be6SWen Congyang 19250f1cfe9dSToshi Kani static int check_cpu_on_node(pg_data_t *pgdat) 192660a5a19eSTang Chen { 192760a5a19eSTang Chen int cpu; 192860a5a19eSTang Chen 192960a5a19eSTang Chen for_each_present_cpu(cpu) { 193060a5a19eSTang Chen if (cpu_to_node(cpu) == pgdat->node_id) 193160a5a19eSTang Chen /* 193260a5a19eSTang Chen * the cpu on this node isn't removed, and we can't 193360a5a19eSTang Chen * offline this node. 193460a5a19eSTang Chen */ 193560a5a19eSTang Chen return -EBUSY; 193660a5a19eSTang Chen } 193760a5a19eSTang Chen 193860a5a19eSTang Chen return 0; 193960a5a19eSTang Chen } 194060a5a19eSTang Chen 19410f1cfe9dSToshi Kani static void unmap_cpu_on_node(pg_data_t *pgdat) 1942e13fe869SWen Congyang { 1943e13fe869SWen Congyang #ifdef CONFIG_ACPI_NUMA 1944e13fe869SWen Congyang int cpu; 1945e13fe869SWen Congyang 1946e13fe869SWen Congyang for_each_possible_cpu(cpu) 1947e13fe869SWen Congyang if (cpu_to_node(cpu) == pgdat->node_id) 1948e13fe869SWen Congyang numa_clear_node(cpu); 1949e13fe869SWen Congyang #endif 1950e13fe869SWen Congyang } 1951e13fe869SWen Congyang 19520f1cfe9dSToshi Kani static int check_and_unmap_cpu_on_node(pg_data_t *pgdat) 1953e13fe869SWen Congyang { 19540f1cfe9dSToshi Kani int ret; 1955e13fe869SWen Congyang 19560f1cfe9dSToshi Kani ret = check_cpu_on_node(pgdat); 1957e13fe869SWen Congyang if (ret) 1958e13fe869SWen Congyang return ret; 1959e13fe869SWen Congyang 1960e13fe869SWen Congyang /* 1961e13fe869SWen Congyang * the node will be offlined when we come here, so we can clear 1962e13fe869SWen Congyang * the cpu_to_node() now. 1963e13fe869SWen Congyang */ 1964e13fe869SWen Congyang 19650f1cfe9dSToshi Kani unmap_cpu_on_node(pgdat); 1966e13fe869SWen Congyang return 0; 1967e13fe869SWen Congyang } 1968e13fe869SWen Congyang 19690f1cfe9dSToshi Kani /** 19700f1cfe9dSToshi Kani * try_offline_node 19710f1cfe9dSToshi Kani * 19720f1cfe9dSToshi Kani * Offline a node if all memory sections and cpus of the node are removed. 19730f1cfe9dSToshi Kani * 19740f1cfe9dSToshi Kani * NOTE: The caller must call lock_device_hotplug() to serialize hotplug 19750f1cfe9dSToshi Kani * and online/offline operations before this call. 19760f1cfe9dSToshi Kani */ 197790b30cdcSWen Congyang void try_offline_node(int nid) 197860a5a19eSTang Chen { 1979d822b86aSWen Congyang pg_data_t *pgdat = NODE_DATA(nid); 1980d822b86aSWen Congyang unsigned long start_pfn = pgdat->node_start_pfn; 1981d822b86aSWen Congyang unsigned long end_pfn = start_pfn + pgdat->node_spanned_pages; 198260a5a19eSTang Chen unsigned long pfn; 1983d822b86aSWen Congyang int i; 198460a5a19eSTang Chen 198560a5a19eSTang Chen for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) { 198660a5a19eSTang Chen unsigned long section_nr = pfn_to_section_nr(pfn); 198760a5a19eSTang Chen 198860a5a19eSTang Chen if (!present_section_nr(section_nr)) 198960a5a19eSTang Chen continue; 199060a5a19eSTang Chen 199160a5a19eSTang Chen if (pfn_to_nid(pfn) != nid) 199260a5a19eSTang Chen continue; 199360a5a19eSTang Chen 199460a5a19eSTang Chen /* 199560a5a19eSTang Chen * some memory sections of this node are not removed, and we 199660a5a19eSTang Chen * can't offline node now. 199760a5a19eSTang Chen */ 199860a5a19eSTang Chen return; 199960a5a19eSTang Chen } 200060a5a19eSTang Chen 20010f1cfe9dSToshi Kani if (check_and_unmap_cpu_on_node(pgdat)) 200260a5a19eSTang Chen return; 200360a5a19eSTang Chen 200460a5a19eSTang Chen /* 200560a5a19eSTang Chen * all memory/cpu of this node are removed, we can offline this 200660a5a19eSTang Chen * node now. 200760a5a19eSTang Chen */ 200860a5a19eSTang Chen node_set_offline(nid); 200960a5a19eSTang Chen unregister_one_node(nid); 2010d822b86aSWen Congyang 2011d822b86aSWen Congyang /* free waittable in each zone */ 2012d822b86aSWen Congyang for (i = 0; i < MAX_NR_ZONES; i++) { 2013d822b86aSWen Congyang struct zone *zone = pgdat->node_zones + i; 2014d822b86aSWen Congyang 2015ca4b3f30SJianguo Wu /* 2016ca4b3f30SJianguo Wu * wait_table may be allocated from boot memory, 2017ca4b3f30SJianguo Wu * here only free if it's allocated by vmalloc. 2018ca4b3f30SJianguo Wu */ 201985bd8399SGu Zheng if (is_vmalloc_addr(zone->wait_table)) { 2020d822b86aSWen Congyang vfree(zone->wait_table); 202185bd8399SGu Zheng zone->wait_table = NULL; 202285bd8399SGu Zheng } 2023d822b86aSWen Congyang } 202460a5a19eSTang Chen } 202590b30cdcSWen Congyang EXPORT_SYMBOL(try_offline_node); 202660a5a19eSTang Chen 20270f1cfe9dSToshi Kani /** 20280f1cfe9dSToshi Kani * remove_memory 20290f1cfe9dSToshi Kani * 20300f1cfe9dSToshi Kani * NOTE: The caller must call lock_device_hotplug() to serialize hotplug 20310f1cfe9dSToshi Kani * and online/offline operations before this call, as required by 20320f1cfe9dSToshi Kani * try_offline_node(). 20330f1cfe9dSToshi Kani */ 2034242831ebSRafael J. Wysocki void __ref remove_memory(int nid, u64 start, u64 size) 2035bbc76be6SWen Congyang { 2036242831ebSRafael J. Wysocki int ret; 2037993c1aadSWen Congyang 203827356f54SToshi Kani BUG_ON(check_hotplug_memory_range(start, size)); 203927356f54SToshi Kani 2040bfc8c901SVladimir Davydov mem_hotplug_begin(); 20416677e3eaSYasuaki Ishimatsu 20426677e3eaSYasuaki Ishimatsu /* 2043242831ebSRafael J. Wysocki * All memory blocks must be offlined before removing memory. Check 2044242831ebSRafael J. Wysocki * whether all memory blocks in question are offline and trigger a BUG() 2045242831ebSRafael J. Wysocki * if this is not the case. 20466677e3eaSYasuaki Ishimatsu */ 2047242831ebSRafael J. Wysocki ret = walk_memory_range(PFN_DOWN(start), PFN_UP(start + size - 1), NULL, 2048d6de9d53SXishi Qiu check_memblock_offlined_cb); 2049bfc8c901SVladimir Davydov if (ret) 2050242831ebSRafael J. Wysocki BUG(); 20516677e3eaSYasuaki Ishimatsu 205246c66c4bSYasuaki Ishimatsu /* remove memmap entry */ 205346c66c4bSYasuaki Ishimatsu firmware_map_remove(start, start + size, "System RAM"); 2054f9126ab9SXishi Qiu memblock_free(start, size); 2055f9126ab9SXishi Qiu memblock_remove(start, size); 205646c66c4bSYasuaki Ishimatsu 205724d335caSWen Congyang arch_remove_memory(start, size); 205824d335caSWen Congyang 205960a5a19eSTang Chen try_offline_node(nid); 206060a5a19eSTang Chen 2061bfc8c901SVladimir Davydov mem_hotplug_done(); 206271088785SBadari Pulavarty } 206371088785SBadari Pulavarty EXPORT_SYMBOL_GPL(remove_memory); 2064aba6efc4SRafael J. Wysocki #endif /* CONFIG_MEMORY_HOTREMOVE */ 2065