1 /* 2 * linux/mm/memory_hotplug.c 3 * 4 * Copyright (C) 5 */ 6 7 #include <linux/stddef.h> 8 #include <linux/mm.h> 9 #include <linux/swap.h> 10 #include <linux/interrupt.h> 11 #include <linux/pagemap.h> 12 #include <linux/compiler.h> 13 #include <linux/export.h> 14 #include <linux/pagevec.h> 15 #include <linux/writeback.h> 16 #include <linux/slab.h> 17 #include <linux/sysctl.h> 18 #include <linux/cpu.h> 19 #include <linux/memory.h> 20 #include <linux/memremap.h> 21 #include <linux/memory_hotplug.h> 22 #include <linux/highmem.h> 23 #include <linux/vmalloc.h> 24 #include <linux/ioport.h> 25 #include <linux/delay.h> 26 #include <linux/migrate.h> 27 #include <linux/page-isolation.h> 28 #include <linux/pfn.h> 29 #include <linux/suspend.h> 30 #include <linux/mm_inline.h> 31 #include <linux/firmware-map.h> 32 #include <linux/stop_machine.h> 33 #include <linux/hugetlb.h> 34 #include <linux/memblock.h> 35 #include <linux/bootmem.h> 36 #include <linux/compaction.h> 37 38 #include <asm/tlbflush.h> 39 40 #include "internal.h" 41 42 /* 43 * online_page_callback contains pointer to current page onlining function. 44 * Initially it is generic_online_page(). If it is required it could be 45 * changed by calling set_online_page_callback() for callback registration 46 * and restore_online_page_callback() for generic callback restore. 47 */ 48 49 static void generic_online_page(struct page *page); 50 51 static online_page_callback_t online_page_callback = generic_online_page; 52 static DEFINE_MUTEX(online_page_callback_lock); 53 54 /* The same as the cpu_hotplug lock, but for memory hotplug. */ 55 static struct { 56 struct task_struct *active_writer; 57 struct mutex lock; /* Synchronizes accesses to refcount, */ 58 /* 59 * Also blocks the new readers during 60 * an ongoing mem hotplug operation. 61 */ 62 int refcount; 63 64 #ifdef CONFIG_DEBUG_LOCK_ALLOC 65 struct lockdep_map dep_map; 66 #endif 67 } mem_hotplug = { 68 .active_writer = NULL, 69 .lock = __MUTEX_INITIALIZER(mem_hotplug.lock), 70 .refcount = 0, 71 #ifdef CONFIG_DEBUG_LOCK_ALLOC 72 .dep_map = {.name = "mem_hotplug.lock" }, 73 #endif 74 }; 75 76 /* Lockdep annotations for get/put_online_mems() and mem_hotplug_begin/end() */ 77 #define memhp_lock_acquire_read() lock_map_acquire_read(&mem_hotplug.dep_map) 78 #define memhp_lock_acquire() lock_map_acquire(&mem_hotplug.dep_map) 79 #define memhp_lock_release() lock_map_release(&mem_hotplug.dep_map) 80 81 bool memhp_auto_online; 82 EXPORT_SYMBOL_GPL(memhp_auto_online); 83 84 void get_online_mems(void) 85 { 86 might_sleep(); 87 if (mem_hotplug.active_writer == current) 88 return; 89 memhp_lock_acquire_read(); 90 mutex_lock(&mem_hotplug.lock); 91 mem_hotplug.refcount++; 92 mutex_unlock(&mem_hotplug.lock); 93 94 } 95 96 void put_online_mems(void) 97 { 98 if (mem_hotplug.active_writer == current) 99 return; 100 mutex_lock(&mem_hotplug.lock); 101 102 if (WARN_ON(!mem_hotplug.refcount)) 103 mem_hotplug.refcount++; /* try to fix things up */ 104 105 if (!--mem_hotplug.refcount && unlikely(mem_hotplug.active_writer)) 106 wake_up_process(mem_hotplug.active_writer); 107 mutex_unlock(&mem_hotplug.lock); 108 memhp_lock_release(); 109 110 } 111 112 void mem_hotplug_begin(void) 113 { 114 mem_hotplug.active_writer = current; 115 116 memhp_lock_acquire(); 117 for (;;) { 118 mutex_lock(&mem_hotplug.lock); 119 if (likely(!mem_hotplug.refcount)) 120 break; 121 __set_current_state(TASK_UNINTERRUPTIBLE); 122 mutex_unlock(&mem_hotplug.lock); 123 schedule(); 124 } 125 } 126 127 void mem_hotplug_done(void) 128 { 129 mem_hotplug.active_writer = NULL; 130 mutex_unlock(&mem_hotplug.lock); 131 memhp_lock_release(); 132 } 133 134 /* add this memory to iomem resource */ 135 static struct resource *register_memory_resource(u64 start, u64 size) 136 { 137 struct resource *res; 138 res = kzalloc(sizeof(struct resource), GFP_KERNEL); 139 if (!res) 140 return ERR_PTR(-ENOMEM); 141 142 res->name = "System RAM"; 143 res->start = start; 144 res->end = start + size - 1; 145 res->flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY; 146 if (request_resource(&iomem_resource, res) < 0) { 147 pr_debug("System RAM resource %pR cannot be added\n", res); 148 kfree(res); 149 return ERR_PTR(-EEXIST); 150 } 151 return res; 152 } 153 154 static void release_memory_resource(struct resource *res) 155 { 156 if (!res) 157 return; 158 release_resource(res); 159 kfree(res); 160 return; 161 } 162 163 #ifdef CONFIG_MEMORY_HOTPLUG_SPARSE 164 void get_page_bootmem(unsigned long info, struct page *page, 165 unsigned long type) 166 { 167 page->lru.next = (struct list_head *) type; 168 SetPagePrivate(page); 169 set_page_private(page, info); 170 page_ref_inc(page); 171 } 172 173 void put_page_bootmem(struct page *page) 174 { 175 unsigned long type; 176 177 type = (unsigned long) page->lru.next; 178 BUG_ON(type < MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE || 179 type > MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE); 180 181 if (page_ref_dec_return(page) == 1) { 182 ClearPagePrivate(page); 183 set_page_private(page, 0); 184 INIT_LIST_HEAD(&page->lru); 185 free_reserved_page(page); 186 } 187 } 188 189 #ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE 190 #ifndef CONFIG_SPARSEMEM_VMEMMAP 191 static void register_page_bootmem_info_section(unsigned long start_pfn) 192 { 193 unsigned long *usemap, mapsize, section_nr, i; 194 struct mem_section *ms; 195 struct page *page, *memmap; 196 197 section_nr = pfn_to_section_nr(start_pfn); 198 ms = __nr_to_section(section_nr); 199 200 /* Get section's memmap address */ 201 memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr); 202 203 /* 204 * Get page for the memmap's phys address 205 * XXX: need more consideration for sparse_vmemmap... 206 */ 207 page = virt_to_page(memmap); 208 mapsize = sizeof(struct page) * PAGES_PER_SECTION; 209 mapsize = PAGE_ALIGN(mapsize) >> PAGE_SHIFT; 210 211 /* remember memmap's page */ 212 for (i = 0; i < mapsize; i++, page++) 213 get_page_bootmem(section_nr, page, SECTION_INFO); 214 215 usemap = __nr_to_section(section_nr)->pageblock_flags; 216 page = virt_to_page(usemap); 217 218 mapsize = PAGE_ALIGN(usemap_size()) >> PAGE_SHIFT; 219 220 for (i = 0; i < mapsize; i++, page++) 221 get_page_bootmem(section_nr, page, MIX_SECTION_INFO); 222 223 } 224 #else /* CONFIG_SPARSEMEM_VMEMMAP */ 225 static void register_page_bootmem_info_section(unsigned long start_pfn) 226 { 227 unsigned long *usemap, mapsize, section_nr, i; 228 struct mem_section *ms; 229 struct page *page, *memmap; 230 231 if (!pfn_valid(start_pfn)) 232 return; 233 234 section_nr = pfn_to_section_nr(start_pfn); 235 ms = __nr_to_section(section_nr); 236 237 memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr); 238 239 register_page_bootmem_memmap(section_nr, memmap, PAGES_PER_SECTION); 240 241 usemap = __nr_to_section(section_nr)->pageblock_flags; 242 page = virt_to_page(usemap); 243 244 mapsize = PAGE_ALIGN(usemap_size()) >> PAGE_SHIFT; 245 246 for (i = 0; i < mapsize; i++, page++) 247 get_page_bootmem(section_nr, page, MIX_SECTION_INFO); 248 } 249 #endif /* !CONFIG_SPARSEMEM_VMEMMAP */ 250 251 void register_page_bootmem_info_node(struct pglist_data *pgdat) 252 { 253 unsigned long i, pfn, end_pfn, nr_pages; 254 int node = pgdat->node_id; 255 struct page *page; 256 struct zone *zone; 257 258 nr_pages = PAGE_ALIGN(sizeof(struct pglist_data)) >> PAGE_SHIFT; 259 page = virt_to_page(pgdat); 260 261 for (i = 0; i < nr_pages; i++, page++) 262 get_page_bootmem(node, page, NODE_INFO); 263 264 zone = &pgdat->node_zones[0]; 265 for (; zone < pgdat->node_zones + MAX_NR_ZONES - 1; zone++) { 266 if (zone_is_initialized(zone)) { 267 nr_pages = zone->wait_table_hash_nr_entries 268 * sizeof(wait_queue_head_t); 269 nr_pages = PAGE_ALIGN(nr_pages) >> PAGE_SHIFT; 270 page = virt_to_page(zone->wait_table); 271 272 for (i = 0; i < nr_pages; i++, page++) 273 get_page_bootmem(node, page, NODE_INFO); 274 } 275 } 276 277 pfn = pgdat->node_start_pfn; 278 end_pfn = pgdat_end_pfn(pgdat); 279 280 /* register section info */ 281 for (; pfn < end_pfn; pfn += PAGES_PER_SECTION) { 282 /* 283 * Some platforms can assign the same pfn to multiple nodes - on 284 * node0 as well as nodeN. To avoid registering a pfn against 285 * multiple nodes we check that this pfn does not already 286 * reside in some other nodes. 287 */ 288 if (pfn_valid(pfn) && (pfn_to_nid(pfn) == node)) 289 register_page_bootmem_info_section(pfn); 290 } 291 } 292 #endif /* CONFIG_HAVE_BOOTMEM_INFO_NODE */ 293 294 static void __meminit grow_zone_span(struct zone *zone, unsigned long start_pfn, 295 unsigned long end_pfn) 296 { 297 unsigned long old_zone_end_pfn; 298 299 zone_span_writelock(zone); 300 301 old_zone_end_pfn = zone_end_pfn(zone); 302 if (zone_is_empty(zone) || start_pfn < zone->zone_start_pfn) 303 zone->zone_start_pfn = start_pfn; 304 305 zone->spanned_pages = max(old_zone_end_pfn, end_pfn) - 306 zone->zone_start_pfn; 307 308 zone_span_writeunlock(zone); 309 } 310 311 static void resize_zone(struct zone *zone, unsigned long start_pfn, 312 unsigned long end_pfn) 313 { 314 zone_span_writelock(zone); 315 316 if (end_pfn - start_pfn) { 317 zone->zone_start_pfn = start_pfn; 318 zone->spanned_pages = end_pfn - start_pfn; 319 } else { 320 /* 321 * make it consist as free_area_init_core(), 322 * if spanned_pages = 0, then keep start_pfn = 0 323 */ 324 zone->zone_start_pfn = 0; 325 zone->spanned_pages = 0; 326 } 327 328 zone_span_writeunlock(zone); 329 } 330 331 static void fix_zone_id(struct zone *zone, unsigned long start_pfn, 332 unsigned long end_pfn) 333 { 334 enum zone_type zid = zone_idx(zone); 335 int nid = zone->zone_pgdat->node_id; 336 unsigned long pfn; 337 338 for (pfn = start_pfn; pfn < end_pfn; pfn++) 339 set_page_links(pfn_to_page(pfn), zid, nid, pfn); 340 } 341 342 /* Can fail with -ENOMEM from allocating a wait table with vmalloc() or 343 * alloc_bootmem_node_nopanic()/memblock_virt_alloc_node_nopanic() */ 344 static int __ref ensure_zone_is_initialized(struct zone *zone, 345 unsigned long start_pfn, unsigned long num_pages) 346 { 347 if (!zone_is_initialized(zone)) 348 return init_currently_empty_zone(zone, start_pfn, num_pages); 349 350 return 0; 351 } 352 353 static int __meminit move_pfn_range_left(struct zone *z1, struct zone *z2, 354 unsigned long start_pfn, unsigned long end_pfn) 355 { 356 int ret; 357 unsigned long flags; 358 unsigned long z1_start_pfn; 359 360 ret = ensure_zone_is_initialized(z1, start_pfn, end_pfn - start_pfn); 361 if (ret) 362 return ret; 363 364 pgdat_resize_lock(z1->zone_pgdat, &flags); 365 366 /* can't move pfns which are higher than @z2 */ 367 if (end_pfn > zone_end_pfn(z2)) 368 goto out_fail; 369 /* the move out part must be at the left most of @z2 */ 370 if (start_pfn > z2->zone_start_pfn) 371 goto out_fail; 372 /* must included/overlap */ 373 if (end_pfn <= z2->zone_start_pfn) 374 goto out_fail; 375 376 /* use start_pfn for z1's start_pfn if z1 is empty */ 377 if (!zone_is_empty(z1)) 378 z1_start_pfn = z1->zone_start_pfn; 379 else 380 z1_start_pfn = start_pfn; 381 382 resize_zone(z1, z1_start_pfn, end_pfn); 383 resize_zone(z2, end_pfn, zone_end_pfn(z2)); 384 385 pgdat_resize_unlock(z1->zone_pgdat, &flags); 386 387 fix_zone_id(z1, start_pfn, end_pfn); 388 389 return 0; 390 out_fail: 391 pgdat_resize_unlock(z1->zone_pgdat, &flags); 392 return -1; 393 } 394 395 static int __meminit move_pfn_range_right(struct zone *z1, struct zone *z2, 396 unsigned long start_pfn, unsigned long end_pfn) 397 { 398 int ret; 399 unsigned long flags; 400 unsigned long z2_end_pfn; 401 402 ret = ensure_zone_is_initialized(z2, start_pfn, end_pfn - start_pfn); 403 if (ret) 404 return ret; 405 406 pgdat_resize_lock(z1->zone_pgdat, &flags); 407 408 /* can't move pfns which are lower than @z1 */ 409 if (z1->zone_start_pfn > start_pfn) 410 goto out_fail; 411 /* the move out part mast at the right most of @z1 */ 412 if (zone_end_pfn(z1) > end_pfn) 413 goto out_fail; 414 /* must included/overlap */ 415 if (start_pfn >= zone_end_pfn(z1)) 416 goto out_fail; 417 418 /* use end_pfn for z2's end_pfn if z2 is empty */ 419 if (!zone_is_empty(z2)) 420 z2_end_pfn = zone_end_pfn(z2); 421 else 422 z2_end_pfn = end_pfn; 423 424 resize_zone(z1, z1->zone_start_pfn, start_pfn); 425 resize_zone(z2, start_pfn, z2_end_pfn); 426 427 pgdat_resize_unlock(z1->zone_pgdat, &flags); 428 429 fix_zone_id(z2, start_pfn, end_pfn); 430 431 return 0; 432 out_fail: 433 pgdat_resize_unlock(z1->zone_pgdat, &flags); 434 return -1; 435 } 436 437 static void __meminit grow_pgdat_span(struct pglist_data *pgdat, unsigned long start_pfn, 438 unsigned long end_pfn) 439 { 440 unsigned long old_pgdat_end_pfn = pgdat_end_pfn(pgdat); 441 442 if (!pgdat->node_spanned_pages || start_pfn < pgdat->node_start_pfn) 443 pgdat->node_start_pfn = start_pfn; 444 445 pgdat->node_spanned_pages = max(old_pgdat_end_pfn, end_pfn) - 446 pgdat->node_start_pfn; 447 } 448 449 static int __meminit __add_zone(struct zone *zone, unsigned long phys_start_pfn) 450 { 451 struct pglist_data *pgdat = zone->zone_pgdat; 452 int nr_pages = PAGES_PER_SECTION; 453 int nid = pgdat->node_id; 454 int zone_type; 455 unsigned long flags, pfn; 456 int ret; 457 458 zone_type = zone - pgdat->node_zones; 459 ret = ensure_zone_is_initialized(zone, phys_start_pfn, nr_pages); 460 if (ret) 461 return ret; 462 463 pgdat_resize_lock(zone->zone_pgdat, &flags); 464 grow_zone_span(zone, phys_start_pfn, phys_start_pfn + nr_pages); 465 grow_pgdat_span(zone->zone_pgdat, phys_start_pfn, 466 phys_start_pfn + nr_pages); 467 pgdat_resize_unlock(zone->zone_pgdat, &flags); 468 memmap_init_zone(nr_pages, nid, zone_type, 469 phys_start_pfn, MEMMAP_HOTPLUG); 470 471 /* online_page_range is called later and expects pages reserved */ 472 for (pfn = phys_start_pfn; pfn < phys_start_pfn + nr_pages; pfn++) { 473 if (!pfn_valid(pfn)) 474 continue; 475 476 SetPageReserved(pfn_to_page(pfn)); 477 } 478 return 0; 479 } 480 481 static int __meminit __add_section(int nid, struct zone *zone, 482 unsigned long phys_start_pfn) 483 { 484 int ret; 485 486 if (pfn_valid(phys_start_pfn)) 487 return -EEXIST; 488 489 ret = sparse_add_one_section(zone, phys_start_pfn); 490 491 if (ret < 0) 492 return ret; 493 494 ret = __add_zone(zone, phys_start_pfn); 495 496 if (ret < 0) 497 return ret; 498 499 return register_new_memory(nid, __pfn_to_section(phys_start_pfn)); 500 } 501 502 /* 503 * Reasonably generic function for adding memory. It is 504 * expected that archs that support memory hotplug will 505 * call this function after deciding the zone to which to 506 * add the new pages. 507 */ 508 int __ref __add_pages(int nid, struct zone *zone, unsigned long phys_start_pfn, 509 unsigned long nr_pages) 510 { 511 unsigned long i; 512 int err = 0; 513 int start_sec, end_sec; 514 struct vmem_altmap *altmap; 515 516 clear_zone_contiguous(zone); 517 518 /* during initialize mem_map, align hot-added range to section */ 519 start_sec = pfn_to_section_nr(phys_start_pfn); 520 end_sec = pfn_to_section_nr(phys_start_pfn + nr_pages - 1); 521 522 altmap = to_vmem_altmap((unsigned long) pfn_to_page(phys_start_pfn)); 523 if (altmap) { 524 /* 525 * Validate altmap is within bounds of the total request 526 */ 527 if (altmap->base_pfn != phys_start_pfn 528 || vmem_altmap_offset(altmap) > nr_pages) { 529 pr_warn_once("memory add fail, invalid altmap\n"); 530 err = -EINVAL; 531 goto out; 532 } 533 altmap->alloc = 0; 534 } 535 536 for (i = start_sec; i <= end_sec; i++) { 537 err = __add_section(nid, zone, section_nr_to_pfn(i)); 538 539 /* 540 * EEXIST is finally dealt with by ioresource collision 541 * check. see add_memory() => register_memory_resource() 542 * Warning will be printed if there is collision. 543 */ 544 if (err && (err != -EEXIST)) 545 break; 546 err = 0; 547 } 548 vmemmap_populate_print_last(); 549 out: 550 set_zone_contiguous(zone); 551 return err; 552 } 553 EXPORT_SYMBOL_GPL(__add_pages); 554 555 #ifdef CONFIG_MEMORY_HOTREMOVE 556 /* find the smallest valid pfn in the range [start_pfn, end_pfn) */ 557 static int find_smallest_section_pfn(int nid, struct zone *zone, 558 unsigned long start_pfn, 559 unsigned long end_pfn) 560 { 561 struct mem_section *ms; 562 563 for (; start_pfn < end_pfn; start_pfn += PAGES_PER_SECTION) { 564 ms = __pfn_to_section(start_pfn); 565 566 if (unlikely(!valid_section(ms))) 567 continue; 568 569 if (unlikely(pfn_to_nid(start_pfn) != nid)) 570 continue; 571 572 if (zone && zone != page_zone(pfn_to_page(start_pfn))) 573 continue; 574 575 return start_pfn; 576 } 577 578 return 0; 579 } 580 581 /* find the biggest valid pfn in the range [start_pfn, end_pfn). */ 582 static int find_biggest_section_pfn(int nid, struct zone *zone, 583 unsigned long start_pfn, 584 unsigned long end_pfn) 585 { 586 struct mem_section *ms; 587 unsigned long pfn; 588 589 /* pfn is the end pfn of a memory section. */ 590 pfn = end_pfn - 1; 591 for (; pfn >= start_pfn; pfn -= PAGES_PER_SECTION) { 592 ms = __pfn_to_section(pfn); 593 594 if (unlikely(!valid_section(ms))) 595 continue; 596 597 if (unlikely(pfn_to_nid(pfn) != nid)) 598 continue; 599 600 if (zone && zone != page_zone(pfn_to_page(pfn))) 601 continue; 602 603 return pfn; 604 } 605 606 return 0; 607 } 608 609 static void shrink_zone_span(struct zone *zone, unsigned long start_pfn, 610 unsigned long end_pfn) 611 { 612 unsigned long zone_start_pfn = zone->zone_start_pfn; 613 unsigned long z = zone_end_pfn(zone); /* zone_end_pfn namespace clash */ 614 unsigned long zone_end_pfn = z; 615 unsigned long pfn; 616 struct mem_section *ms; 617 int nid = zone_to_nid(zone); 618 619 zone_span_writelock(zone); 620 if (zone_start_pfn == start_pfn) { 621 /* 622 * If the section is smallest section in the zone, it need 623 * shrink zone->zone_start_pfn and zone->zone_spanned_pages. 624 * In this case, we find second smallest valid mem_section 625 * for shrinking zone. 626 */ 627 pfn = find_smallest_section_pfn(nid, zone, end_pfn, 628 zone_end_pfn); 629 if (pfn) { 630 zone->zone_start_pfn = pfn; 631 zone->spanned_pages = zone_end_pfn - pfn; 632 } 633 } else if (zone_end_pfn == end_pfn) { 634 /* 635 * If the section is biggest section in the zone, it need 636 * shrink zone->spanned_pages. 637 * In this case, we find second biggest valid mem_section for 638 * shrinking zone. 639 */ 640 pfn = find_biggest_section_pfn(nid, zone, zone_start_pfn, 641 start_pfn); 642 if (pfn) 643 zone->spanned_pages = pfn - zone_start_pfn + 1; 644 } 645 646 /* 647 * The section is not biggest or smallest mem_section in the zone, it 648 * only creates a hole in the zone. So in this case, we need not 649 * change the zone. But perhaps, the zone has only hole data. Thus 650 * it check the zone has only hole or not. 651 */ 652 pfn = zone_start_pfn; 653 for (; pfn < zone_end_pfn; pfn += PAGES_PER_SECTION) { 654 ms = __pfn_to_section(pfn); 655 656 if (unlikely(!valid_section(ms))) 657 continue; 658 659 if (page_zone(pfn_to_page(pfn)) != zone) 660 continue; 661 662 /* If the section is current section, it continues the loop */ 663 if (start_pfn == pfn) 664 continue; 665 666 /* If we find valid section, we have nothing to do */ 667 zone_span_writeunlock(zone); 668 return; 669 } 670 671 /* The zone has no valid section */ 672 zone->zone_start_pfn = 0; 673 zone->spanned_pages = 0; 674 zone_span_writeunlock(zone); 675 } 676 677 static void shrink_pgdat_span(struct pglist_data *pgdat, 678 unsigned long start_pfn, unsigned long end_pfn) 679 { 680 unsigned long pgdat_start_pfn = pgdat->node_start_pfn; 681 unsigned long p = pgdat_end_pfn(pgdat); /* pgdat_end_pfn namespace clash */ 682 unsigned long pgdat_end_pfn = p; 683 unsigned long pfn; 684 struct mem_section *ms; 685 int nid = pgdat->node_id; 686 687 if (pgdat_start_pfn == start_pfn) { 688 /* 689 * If the section is smallest section in the pgdat, it need 690 * shrink pgdat->node_start_pfn and pgdat->node_spanned_pages. 691 * In this case, we find second smallest valid mem_section 692 * for shrinking zone. 693 */ 694 pfn = find_smallest_section_pfn(nid, NULL, end_pfn, 695 pgdat_end_pfn); 696 if (pfn) { 697 pgdat->node_start_pfn = pfn; 698 pgdat->node_spanned_pages = pgdat_end_pfn - pfn; 699 } 700 } else if (pgdat_end_pfn == end_pfn) { 701 /* 702 * If the section is biggest section in the pgdat, it need 703 * shrink pgdat->node_spanned_pages. 704 * In this case, we find second biggest valid mem_section for 705 * shrinking zone. 706 */ 707 pfn = find_biggest_section_pfn(nid, NULL, pgdat_start_pfn, 708 start_pfn); 709 if (pfn) 710 pgdat->node_spanned_pages = pfn - pgdat_start_pfn + 1; 711 } 712 713 /* 714 * If the section is not biggest or smallest mem_section in the pgdat, 715 * it only creates a hole in the pgdat. So in this case, we need not 716 * change the pgdat. 717 * But perhaps, the pgdat has only hole data. Thus it check the pgdat 718 * has only hole or not. 719 */ 720 pfn = pgdat_start_pfn; 721 for (; pfn < pgdat_end_pfn; pfn += PAGES_PER_SECTION) { 722 ms = __pfn_to_section(pfn); 723 724 if (unlikely(!valid_section(ms))) 725 continue; 726 727 if (pfn_to_nid(pfn) != nid) 728 continue; 729 730 /* If the section is current section, it continues the loop */ 731 if (start_pfn == pfn) 732 continue; 733 734 /* If we find valid section, we have nothing to do */ 735 return; 736 } 737 738 /* The pgdat has no valid section */ 739 pgdat->node_start_pfn = 0; 740 pgdat->node_spanned_pages = 0; 741 } 742 743 static void __remove_zone(struct zone *zone, unsigned long start_pfn) 744 { 745 struct pglist_data *pgdat = zone->zone_pgdat; 746 int nr_pages = PAGES_PER_SECTION; 747 int zone_type; 748 unsigned long flags; 749 750 zone_type = zone - pgdat->node_zones; 751 752 pgdat_resize_lock(zone->zone_pgdat, &flags); 753 shrink_zone_span(zone, start_pfn, start_pfn + nr_pages); 754 shrink_pgdat_span(pgdat, start_pfn, start_pfn + nr_pages); 755 pgdat_resize_unlock(zone->zone_pgdat, &flags); 756 } 757 758 static int __remove_section(struct zone *zone, struct mem_section *ms, 759 unsigned long map_offset) 760 { 761 unsigned long start_pfn; 762 int scn_nr; 763 int ret = -EINVAL; 764 765 if (!valid_section(ms)) 766 return ret; 767 768 ret = unregister_memory_section(ms); 769 if (ret) 770 return ret; 771 772 scn_nr = __section_nr(ms); 773 start_pfn = section_nr_to_pfn(scn_nr); 774 __remove_zone(zone, start_pfn); 775 776 sparse_remove_one_section(zone, ms, map_offset); 777 return 0; 778 } 779 780 /** 781 * __remove_pages() - remove sections of pages from a zone 782 * @zone: zone from which pages need to be removed 783 * @phys_start_pfn: starting pageframe (must be aligned to start of a section) 784 * @nr_pages: number of pages to remove (must be multiple of section size) 785 * 786 * Generic helper function to remove section mappings and sysfs entries 787 * for the section of the memory we are removing. Caller needs to make 788 * sure that pages are marked reserved and zones are adjust properly by 789 * calling offline_pages(). 790 */ 791 int __remove_pages(struct zone *zone, unsigned long phys_start_pfn, 792 unsigned long nr_pages) 793 { 794 unsigned long i; 795 unsigned long map_offset = 0; 796 int sections_to_remove, ret = 0; 797 798 /* In the ZONE_DEVICE case device driver owns the memory region */ 799 if (is_dev_zone(zone)) { 800 struct page *page = pfn_to_page(phys_start_pfn); 801 struct vmem_altmap *altmap; 802 803 altmap = to_vmem_altmap((unsigned long) page); 804 if (altmap) 805 map_offset = vmem_altmap_offset(altmap); 806 } else { 807 resource_size_t start, size; 808 809 start = phys_start_pfn << PAGE_SHIFT; 810 size = nr_pages * PAGE_SIZE; 811 812 ret = release_mem_region_adjustable(&iomem_resource, start, 813 size); 814 if (ret) { 815 resource_size_t endres = start + size - 1; 816 817 pr_warn("Unable to release resource <%pa-%pa> (%d)\n", 818 &start, &endres, ret); 819 } 820 } 821 822 clear_zone_contiguous(zone); 823 824 /* 825 * We can only remove entire sections 826 */ 827 BUG_ON(phys_start_pfn & ~PAGE_SECTION_MASK); 828 BUG_ON(nr_pages % PAGES_PER_SECTION); 829 830 sections_to_remove = nr_pages / PAGES_PER_SECTION; 831 for (i = 0; i < sections_to_remove; i++) { 832 unsigned long pfn = phys_start_pfn + i*PAGES_PER_SECTION; 833 834 ret = __remove_section(zone, __pfn_to_section(pfn), map_offset); 835 map_offset = 0; 836 if (ret) 837 break; 838 } 839 840 set_zone_contiguous(zone); 841 842 return ret; 843 } 844 EXPORT_SYMBOL_GPL(__remove_pages); 845 #endif /* CONFIG_MEMORY_HOTREMOVE */ 846 847 int set_online_page_callback(online_page_callback_t callback) 848 { 849 int rc = -EINVAL; 850 851 get_online_mems(); 852 mutex_lock(&online_page_callback_lock); 853 854 if (online_page_callback == generic_online_page) { 855 online_page_callback = callback; 856 rc = 0; 857 } 858 859 mutex_unlock(&online_page_callback_lock); 860 put_online_mems(); 861 862 return rc; 863 } 864 EXPORT_SYMBOL_GPL(set_online_page_callback); 865 866 int restore_online_page_callback(online_page_callback_t callback) 867 { 868 int rc = -EINVAL; 869 870 get_online_mems(); 871 mutex_lock(&online_page_callback_lock); 872 873 if (online_page_callback == callback) { 874 online_page_callback = generic_online_page; 875 rc = 0; 876 } 877 878 mutex_unlock(&online_page_callback_lock); 879 put_online_mems(); 880 881 return rc; 882 } 883 EXPORT_SYMBOL_GPL(restore_online_page_callback); 884 885 void __online_page_set_limits(struct page *page) 886 { 887 } 888 EXPORT_SYMBOL_GPL(__online_page_set_limits); 889 890 void __online_page_increment_counters(struct page *page) 891 { 892 adjust_managed_page_count(page, 1); 893 } 894 EXPORT_SYMBOL_GPL(__online_page_increment_counters); 895 896 void __online_page_free(struct page *page) 897 { 898 __free_reserved_page(page); 899 } 900 EXPORT_SYMBOL_GPL(__online_page_free); 901 902 static void generic_online_page(struct page *page) 903 { 904 __online_page_set_limits(page); 905 __online_page_increment_counters(page); 906 __online_page_free(page); 907 } 908 909 static int online_pages_range(unsigned long start_pfn, unsigned long nr_pages, 910 void *arg) 911 { 912 unsigned long i; 913 unsigned long onlined_pages = *(unsigned long *)arg; 914 struct page *page; 915 if (PageReserved(pfn_to_page(start_pfn))) 916 for (i = 0; i < nr_pages; i++) { 917 page = pfn_to_page(start_pfn + i); 918 (*online_page_callback)(page); 919 onlined_pages++; 920 } 921 *(unsigned long *)arg = onlined_pages; 922 return 0; 923 } 924 925 #ifdef CONFIG_MOVABLE_NODE 926 /* 927 * When CONFIG_MOVABLE_NODE, we permit onlining of a node which doesn't have 928 * normal memory. 929 */ 930 static bool can_online_high_movable(struct zone *zone) 931 { 932 return true; 933 } 934 #else /* CONFIG_MOVABLE_NODE */ 935 /* ensure every online node has NORMAL memory */ 936 static bool can_online_high_movable(struct zone *zone) 937 { 938 return node_state(zone_to_nid(zone), N_NORMAL_MEMORY); 939 } 940 #endif /* CONFIG_MOVABLE_NODE */ 941 942 /* check which state of node_states will be changed when online memory */ 943 static void node_states_check_changes_online(unsigned long nr_pages, 944 struct zone *zone, struct memory_notify *arg) 945 { 946 int nid = zone_to_nid(zone); 947 enum zone_type zone_last = ZONE_NORMAL; 948 949 /* 950 * If we have HIGHMEM or movable node, node_states[N_NORMAL_MEMORY] 951 * contains nodes which have zones of 0...ZONE_NORMAL, 952 * set zone_last to ZONE_NORMAL. 953 * 954 * If we don't have HIGHMEM nor movable node, 955 * node_states[N_NORMAL_MEMORY] contains nodes which have zones of 956 * 0...ZONE_MOVABLE, set zone_last to ZONE_MOVABLE. 957 */ 958 if (N_MEMORY == N_NORMAL_MEMORY) 959 zone_last = ZONE_MOVABLE; 960 961 /* 962 * if the memory to be online is in a zone of 0...zone_last, and 963 * the zones of 0...zone_last don't have memory before online, we will 964 * need to set the node to node_states[N_NORMAL_MEMORY] after 965 * the memory is online. 966 */ 967 if (zone_idx(zone) <= zone_last && !node_state(nid, N_NORMAL_MEMORY)) 968 arg->status_change_nid_normal = nid; 969 else 970 arg->status_change_nid_normal = -1; 971 972 #ifdef CONFIG_HIGHMEM 973 /* 974 * If we have movable node, node_states[N_HIGH_MEMORY] 975 * contains nodes which have zones of 0...ZONE_HIGHMEM, 976 * set zone_last to ZONE_HIGHMEM. 977 * 978 * If we don't have movable node, node_states[N_NORMAL_MEMORY] 979 * contains nodes which have zones of 0...ZONE_MOVABLE, 980 * set zone_last to ZONE_MOVABLE. 981 */ 982 zone_last = ZONE_HIGHMEM; 983 if (N_MEMORY == N_HIGH_MEMORY) 984 zone_last = ZONE_MOVABLE; 985 986 if (zone_idx(zone) <= zone_last && !node_state(nid, N_HIGH_MEMORY)) 987 arg->status_change_nid_high = nid; 988 else 989 arg->status_change_nid_high = -1; 990 #else 991 arg->status_change_nid_high = arg->status_change_nid_normal; 992 #endif 993 994 /* 995 * if the node don't have memory befor online, we will need to 996 * set the node to node_states[N_MEMORY] after the memory 997 * is online. 998 */ 999 if (!node_state(nid, N_MEMORY)) 1000 arg->status_change_nid = nid; 1001 else 1002 arg->status_change_nid = -1; 1003 } 1004 1005 static void node_states_set_node(int node, struct memory_notify *arg) 1006 { 1007 if (arg->status_change_nid_normal >= 0) 1008 node_set_state(node, N_NORMAL_MEMORY); 1009 1010 if (arg->status_change_nid_high >= 0) 1011 node_set_state(node, N_HIGH_MEMORY); 1012 1013 node_set_state(node, N_MEMORY); 1014 } 1015 1016 1017 /* Must be protected by mem_hotplug_begin() */ 1018 int __ref online_pages(unsigned long pfn, unsigned long nr_pages, int online_type) 1019 { 1020 unsigned long flags; 1021 unsigned long onlined_pages = 0; 1022 struct zone *zone; 1023 int need_zonelists_rebuild = 0; 1024 int nid; 1025 int ret; 1026 struct memory_notify arg; 1027 1028 /* 1029 * This doesn't need a lock to do pfn_to_page(). 1030 * The section can't be removed here because of the 1031 * memory_block->state_mutex. 1032 */ 1033 zone = page_zone(pfn_to_page(pfn)); 1034 1035 if ((zone_idx(zone) > ZONE_NORMAL || 1036 online_type == MMOP_ONLINE_MOVABLE) && 1037 !can_online_high_movable(zone)) 1038 return -EINVAL; 1039 1040 if (online_type == MMOP_ONLINE_KERNEL && 1041 zone_idx(zone) == ZONE_MOVABLE) { 1042 if (move_pfn_range_left(zone - 1, zone, pfn, pfn + nr_pages)) 1043 return -EINVAL; 1044 } 1045 if (online_type == MMOP_ONLINE_MOVABLE && 1046 zone_idx(zone) == ZONE_MOVABLE - 1) { 1047 if (move_pfn_range_right(zone, zone + 1, pfn, pfn + nr_pages)) 1048 return -EINVAL; 1049 } 1050 1051 /* Previous code may changed the zone of the pfn range */ 1052 zone = page_zone(pfn_to_page(pfn)); 1053 1054 arg.start_pfn = pfn; 1055 arg.nr_pages = nr_pages; 1056 node_states_check_changes_online(nr_pages, zone, &arg); 1057 1058 nid = zone_to_nid(zone); 1059 1060 ret = memory_notify(MEM_GOING_ONLINE, &arg); 1061 ret = notifier_to_errno(ret); 1062 if (ret) 1063 goto failed_addition; 1064 1065 /* 1066 * If this zone is not populated, then it is not in zonelist. 1067 * This means the page allocator ignores this zone. 1068 * So, zonelist must be updated after online. 1069 */ 1070 mutex_lock(&zonelists_mutex); 1071 if (!populated_zone(zone)) { 1072 need_zonelists_rebuild = 1; 1073 build_all_zonelists(NULL, zone); 1074 } 1075 1076 ret = walk_system_ram_range(pfn, nr_pages, &onlined_pages, 1077 online_pages_range); 1078 if (ret) { 1079 if (need_zonelists_rebuild) 1080 zone_pcp_reset(zone); 1081 mutex_unlock(&zonelists_mutex); 1082 goto failed_addition; 1083 } 1084 1085 zone->present_pages += onlined_pages; 1086 1087 pgdat_resize_lock(zone->zone_pgdat, &flags); 1088 zone->zone_pgdat->node_present_pages += onlined_pages; 1089 pgdat_resize_unlock(zone->zone_pgdat, &flags); 1090 1091 if (onlined_pages) { 1092 node_states_set_node(nid, &arg); 1093 if (need_zonelists_rebuild) 1094 build_all_zonelists(NULL, NULL); 1095 else 1096 zone_pcp_update(zone); 1097 } 1098 1099 mutex_unlock(&zonelists_mutex); 1100 1101 init_per_zone_wmark_min(); 1102 1103 if (onlined_pages) { 1104 kswapd_run(nid); 1105 kcompactd_run(nid); 1106 } 1107 1108 vm_total_pages = nr_free_pagecache_pages(); 1109 1110 writeback_set_ratelimit(); 1111 1112 if (onlined_pages) 1113 memory_notify(MEM_ONLINE, &arg); 1114 return 0; 1115 1116 failed_addition: 1117 pr_debug("online_pages [mem %#010llx-%#010llx] failed\n", 1118 (unsigned long long) pfn << PAGE_SHIFT, 1119 (((unsigned long long) pfn + nr_pages) << PAGE_SHIFT) - 1); 1120 memory_notify(MEM_CANCEL_ONLINE, &arg); 1121 return ret; 1122 } 1123 #endif /* CONFIG_MEMORY_HOTPLUG_SPARSE */ 1124 1125 static void reset_node_present_pages(pg_data_t *pgdat) 1126 { 1127 struct zone *z; 1128 1129 for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++) 1130 z->present_pages = 0; 1131 1132 pgdat->node_present_pages = 0; 1133 } 1134 1135 /* we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG */ 1136 static pg_data_t __ref *hotadd_new_pgdat(int nid, u64 start) 1137 { 1138 struct pglist_data *pgdat; 1139 unsigned long zones_size[MAX_NR_ZONES] = {0}; 1140 unsigned long zholes_size[MAX_NR_ZONES] = {0}; 1141 unsigned long start_pfn = PFN_DOWN(start); 1142 1143 pgdat = NODE_DATA(nid); 1144 if (!pgdat) { 1145 pgdat = arch_alloc_nodedata(nid); 1146 if (!pgdat) 1147 return NULL; 1148 1149 arch_refresh_nodedata(nid, pgdat); 1150 } else { 1151 /* Reset the nr_zones and classzone_idx to 0 before reuse */ 1152 pgdat->nr_zones = 0; 1153 pgdat->classzone_idx = 0; 1154 } 1155 1156 /* we can use NODE_DATA(nid) from here */ 1157 1158 /* init node's zones as empty zones, we don't have any present pages.*/ 1159 free_area_init_node(nid, zones_size, start_pfn, zholes_size); 1160 1161 /* 1162 * The node we allocated has no zone fallback lists. For avoiding 1163 * to access not-initialized zonelist, build here. 1164 */ 1165 mutex_lock(&zonelists_mutex); 1166 build_all_zonelists(pgdat, NULL); 1167 mutex_unlock(&zonelists_mutex); 1168 1169 /* 1170 * zone->managed_pages is set to an approximate value in 1171 * free_area_init_core(), which will cause 1172 * /sys/device/system/node/nodeX/meminfo has wrong data. 1173 * So reset it to 0 before any memory is onlined. 1174 */ 1175 reset_node_managed_pages(pgdat); 1176 1177 /* 1178 * When memory is hot-added, all the memory is in offline state. So 1179 * clear all zones' present_pages because they will be updated in 1180 * online_pages() and offline_pages(). 1181 */ 1182 reset_node_present_pages(pgdat); 1183 1184 return pgdat; 1185 } 1186 1187 static void rollback_node_hotadd(int nid, pg_data_t *pgdat) 1188 { 1189 arch_refresh_nodedata(nid, NULL); 1190 arch_free_nodedata(pgdat); 1191 return; 1192 } 1193 1194 1195 /** 1196 * try_online_node - online a node if offlined 1197 * 1198 * called by cpu_up() to online a node without onlined memory. 1199 */ 1200 int try_online_node(int nid) 1201 { 1202 pg_data_t *pgdat; 1203 int ret; 1204 1205 if (node_online(nid)) 1206 return 0; 1207 1208 mem_hotplug_begin(); 1209 pgdat = hotadd_new_pgdat(nid, 0); 1210 if (!pgdat) { 1211 pr_err("Cannot online node %d due to NULL pgdat\n", nid); 1212 ret = -ENOMEM; 1213 goto out; 1214 } 1215 node_set_online(nid); 1216 ret = register_one_node(nid); 1217 BUG_ON(ret); 1218 1219 if (pgdat->node_zonelists->_zonerefs->zone == NULL) { 1220 mutex_lock(&zonelists_mutex); 1221 build_all_zonelists(NULL, NULL); 1222 mutex_unlock(&zonelists_mutex); 1223 } 1224 1225 out: 1226 mem_hotplug_done(); 1227 return ret; 1228 } 1229 1230 static int check_hotplug_memory_range(u64 start, u64 size) 1231 { 1232 u64 start_pfn = PFN_DOWN(start); 1233 u64 nr_pages = size >> PAGE_SHIFT; 1234 1235 /* Memory range must be aligned with section */ 1236 if ((start_pfn & ~PAGE_SECTION_MASK) || 1237 (nr_pages % PAGES_PER_SECTION) || (!nr_pages)) { 1238 pr_err("Section-unaligned hotplug range: start 0x%llx, size 0x%llx\n", 1239 (unsigned long long)start, 1240 (unsigned long long)size); 1241 return -EINVAL; 1242 } 1243 1244 return 0; 1245 } 1246 1247 /* 1248 * If movable zone has already been setup, newly added memory should be check. 1249 * If its address is higher than movable zone, it should be added as movable. 1250 * Without this check, movable zone may overlap with other zone. 1251 */ 1252 static int should_add_memory_movable(int nid, u64 start, u64 size) 1253 { 1254 unsigned long start_pfn = start >> PAGE_SHIFT; 1255 pg_data_t *pgdat = NODE_DATA(nid); 1256 struct zone *movable_zone = pgdat->node_zones + ZONE_MOVABLE; 1257 1258 if (zone_is_empty(movable_zone)) 1259 return 0; 1260 1261 if (movable_zone->zone_start_pfn <= start_pfn) 1262 return 1; 1263 1264 return 0; 1265 } 1266 1267 int zone_for_memory(int nid, u64 start, u64 size, int zone_default, 1268 bool for_device) 1269 { 1270 #ifdef CONFIG_ZONE_DEVICE 1271 if (for_device) 1272 return ZONE_DEVICE; 1273 #endif 1274 if (should_add_memory_movable(nid, start, size)) 1275 return ZONE_MOVABLE; 1276 1277 return zone_default; 1278 } 1279 1280 static int online_memory_block(struct memory_block *mem, void *arg) 1281 { 1282 return memory_block_change_state(mem, MEM_ONLINE, MEM_OFFLINE); 1283 } 1284 1285 /* we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG */ 1286 int __ref add_memory_resource(int nid, struct resource *res, bool online) 1287 { 1288 u64 start, size; 1289 pg_data_t *pgdat = NULL; 1290 bool new_pgdat; 1291 bool new_node; 1292 int ret; 1293 1294 start = res->start; 1295 size = resource_size(res); 1296 1297 ret = check_hotplug_memory_range(start, size); 1298 if (ret) 1299 return ret; 1300 1301 { /* Stupid hack to suppress address-never-null warning */ 1302 void *p = NODE_DATA(nid); 1303 new_pgdat = !p; 1304 } 1305 1306 mem_hotplug_begin(); 1307 1308 /* 1309 * Add new range to memblock so that when hotadd_new_pgdat() is called 1310 * to allocate new pgdat, get_pfn_range_for_nid() will be able to find 1311 * this new range and calculate total pages correctly. The range will 1312 * be removed at hot-remove time. 1313 */ 1314 memblock_add_node(start, size, nid); 1315 1316 new_node = !node_online(nid); 1317 if (new_node) { 1318 pgdat = hotadd_new_pgdat(nid, start); 1319 ret = -ENOMEM; 1320 if (!pgdat) 1321 goto error; 1322 } 1323 1324 /* call arch's memory hotadd */ 1325 ret = arch_add_memory(nid, start, size, false); 1326 1327 if (ret < 0) 1328 goto error; 1329 1330 /* we online node here. we can't roll back from here. */ 1331 node_set_online(nid); 1332 1333 if (new_node) { 1334 ret = register_one_node(nid); 1335 /* 1336 * If sysfs file of new node can't create, cpu on the node 1337 * can't be hot-added. There is no rollback way now. 1338 * So, check by BUG_ON() to catch it reluctantly.. 1339 */ 1340 BUG_ON(ret); 1341 } 1342 1343 /* create new memmap entry */ 1344 firmware_map_add_hotplug(start, start + size, "System RAM"); 1345 1346 /* online pages if requested */ 1347 if (online) 1348 walk_memory_range(PFN_DOWN(start), PFN_UP(start + size - 1), 1349 NULL, online_memory_block); 1350 1351 goto out; 1352 1353 error: 1354 /* rollback pgdat allocation and others */ 1355 if (new_pgdat) 1356 rollback_node_hotadd(nid, pgdat); 1357 memblock_remove(start, size); 1358 1359 out: 1360 mem_hotplug_done(); 1361 return ret; 1362 } 1363 EXPORT_SYMBOL_GPL(add_memory_resource); 1364 1365 int __ref add_memory(int nid, u64 start, u64 size) 1366 { 1367 struct resource *res; 1368 int ret; 1369 1370 res = register_memory_resource(start, size); 1371 if (IS_ERR(res)) 1372 return PTR_ERR(res); 1373 1374 ret = add_memory_resource(nid, res, memhp_auto_online); 1375 if (ret < 0) 1376 release_memory_resource(res); 1377 return ret; 1378 } 1379 EXPORT_SYMBOL_GPL(add_memory); 1380 1381 #ifdef CONFIG_MEMORY_HOTREMOVE 1382 /* 1383 * A free page on the buddy free lists (not the per-cpu lists) has PageBuddy 1384 * set and the size of the free page is given by page_order(). Using this, 1385 * the function determines if the pageblock contains only free pages. 1386 * Due to buddy contraints, a free page at least the size of a pageblock will 1387 * be located at the start of the pageblock 1388 */ 1389 static inline int pageblock_free(struct page *page) 1390 { 1391 return PageBuddy(page) && page_order(page) >= pageblock_order; 1392 } 1393 1394 /* Return the start of the next active pageblock after a given page */ 1395 static struct page *next_active_pageblock(struct page *page) 1396 { 1397 /* Ensure the starting page is pageblock-aligned */ 1398 BUG_ON(page_to_pfn(page) & (pageblock_nr_pages - 1)); 1399 1400 /* If the entire pageblock is free, move to the end of free page */ 1401 if (pageblock_free(page)) { 1402 int order; 1403 /* be careful. we don't have locks, page_order can be changed.*/ 1404 order = page_order(page); 1405 if ((order < MAX_ORDER) && (order >= pageblock_order)) 1406 return page + (1 << order); 1407 } 1408 1409 return page + pageblock_nr_pages; 1410 } 1411 1412 /* Checks if this range of memory is likely to be hot-removable. */ 1413 int is_mem_section_removable(unsigned long start_pfn, unsigned long nr_pages) 1414 { 1415 struct page *page = pfn_to_page(start_pfn); 1416 struct page *end_page = page + nr_pages; 1417 1418 /* Check the starting page of each pageblock within the range */ 1419 for (; page < end_page; page = next_active_pageblock(page)) { 1420 if (!is_pageblock_removable_nolock(page)) 1421 return 0; 1422 cond_resched(); 1423 } 1424 1425 /* All pageblocks in the memory block are likely to be hot-removable */ 1426 return 1; 1427 } 1428 1429 /* 1430 * Confirm all pages in a range [start, end) is belongs to the same zone. 1431 */ 1432 int test_pages_in_a_zone(unsigned long start_pfn, unsigned long end_pfn) 1433 { 1434 unsigned long pfn, sec_end_pfn; 1435 struct zone *zone = NULL; 1436 struct page *page; 1437 int i; 1438 for (pfn = start_pfn, sec_end_pfn = SECTION_ALIGN_UP(start_pfn); 1439 pfn < end_pfn; 1440 pfn = sec_end_pfn + 1, sec_end_pfn += PAGES_PER_SECTION) { 1441 /* Make sure the memory section is present first */ 1442 if (!present_section_nr(pfn_to_section_nr(pfn))) 1443 continue; 1444 for (; pfn < sec_end_pfn && pfn < end_pfn; 1445 pfn += MAX_ORDER_NR_PAGES) { 1446 i = 0; 1447 /* This is just a CONFIG_HOLES_IN_ZONE check.*/ 1448 while ((i < MAX_ORDER_NR_PAGES) && 1449 !pfn_valid_within(pfn + i)) 1450 i++; 1451 if (i == MAX_ORDER_NR_PAGES) 1452 continue; 1453 page = pfn_to_page(pfn + i); 1454 if (zone && page_zone(page) != zone) 1455 return 0; 1456 zone = page_zone(page); 1457 } 1458 } 1459 return 1; 1460 } 1461 1462 /* 1463 * Scan pfn range [start,end) to find movable/migratable pages (LRU pages 1464 * and hugepages). We scan pfn because it's much easier than scanning over 1465 * linked list. This function returns the pfn of the first found movable 1466 * page if it's found, otherwise 0. 1467 */ 1468 static unsigned long scan_movable_pages(unsigned long start, unsigned long end) 1469 { 1470 unsigned long pfn; 1471 struct page *page; 1472 for (pfn = start; pfn < end; pfn++) { 1473 if (pfn_valid(pfn)) { 1474 page = pfn_to_page(pfn); 1475 if (PageLRU(page)) 1476 return pfn; 1477 if (PageHuge(page)) { 1478 if (page_huge_active(page)) 1479 return pfn; 1480 else 1481 pfn = round_up(pfn + 1, 1482 1 << compound_order(page)) - 1; 1483 } 1484 } 1485 } 1486 return 0; 1487 } 1488 1489 #define NR_OFFLINE_AT_ONCE_PAGES (256) 1490 static int 1491 do_migrate_range(unsigned long start_pfn, unsigned long end_pfn) 1492 { 1493 unsigned long pfn; 1494 struct page *page; 1495 int move_pages = NR_OFFLINE_AT_ONCE_PAGES; 1496 int not_managed = 0; 1497 int ret = 0; 1498 LIST_HEAD(source); 1499 1500 for (pfn = start_pfn; pfn < end_pfn && move_pages > 0; pfn++) { 1501 if (!pfn_valid(pfn)) 1502 continue; 1503 page = pfn_to_page(pfn); 1504 1505 if (PageHuge(page)) { 1506 struct page *head = compound_head(page); 1507 pfn = page_to_pfn(head) + (1<<compound_order(head)) - 1; 1508 if (compound_order(head) > PFN_SECTION_SHIFT) { 1509 ret = -EBUSY; 1510 break; 1511 } 1512 if (isolate_huge_page(page, &source)) 1513 move_pages -= 1 << compound_order(head); 1514 continue; 1515 } 1516 1517 if (!get_page_unless_zero(page)) 1518 continue; 1519 /* 1520 * We can skip free pages. And we can only deal with pages on 1521 * LRU. 1522 */ 1523 ret = isolate_lru_page(page); 1524 if (!ret) { /* Success */ 1525 put_page(page); 1526 list_add_tail(&page->lru, &source); 1527 move_pages--; 1528 inc_zone_page_state(page, NR_ISOLATED_ANON + 1529 page_is_file_cache(page)); 1530 1531 } else { 1532 #ifdef CONFIG_DEBUG_VM 1533 pr_alert("removing pfn %lx from LRU failed\n", pfn); 1534 dump_page(page, "failed to remove from LRU"); 1535 #endif 1536 put_page(page); 1537 /* Because we don't have big zone->lock. we should 1538 check this again here. */ 1539 if (page_count(page)) { 1540 not_managed++; 1541 ret = -EBUSY; 1542 break; 1543 } 1544 } 1545 } 1546 if (!list_empty(&source)) { 1547 if (not_managed) { 1548 putback_movable_pages(&source); 1549 goto out; 1550 } 1551 1552 /* 1553 * alloc_migrate_target should be improooooved!! 1554 * migrate_pages returns # of failed pages. 1555 */ 1556 ret = migrate_pages(&source, alloc_migrate_target, NULL, 0, 1557 MIGRATE_SYNC, MR_MEMORY_HOTPLUG); 1558 if (ret) 1559 putback_movable_pages(&source); 1560 } 1561 out: 1562 return ret; 1563 } 1564 1565 /* 1566 * remove from free_area[] and mark all as Reserved. 1567 */ 1568 static int 1569 offline_isolated_pages_cb(unsigned long start, unsigned long nr_pages, 1570 void *data) 1571 { 1572 __offline_isolated_pages(start, start + nr_pages); 1573 return 0; 1574 } 1575 1576 static void 1577 offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn) 1578 { 1579 walk_system_ram_range(start_pfn, end_pfn - start_pfn, NULL, 1580 offline_isolated_pages_cb); 1581 } 1582 1583 /* 1584 * Check all pages in range, recoreded as memory resource, are isolated. 1585 */ 1586 static int 1587 check_pages_isolated_cb(unsigned long start_pfn, unsigned long nr_pages, 1588 void *data) 1589 { 1590 int ret; 1591 long offlined = *(long *)data; 1592 ret = test_pages_isolated(start_pfn, start_pfn + nr_pages, true); 1593 offlined = nr_pages; 1594 if (!ret) 1595 *(long *)data += offlined; 1596 return ret; 1597 } 1598 1599 static long 1600 check_pages_isolated(unsigned long start_pfn, unsigned long end_pfn) 1601 { 1602 long offlined = 0; 1603 int ret; 1604 1605 ret = walk_system_ram_range(start_pfn, end_pfn - start_pfn, &offlined, 1606 check_pages_isolated_cb); 1607 if (ret < 0) 1608 offlined = (long)ret; 1609 return offlined; 1610 } 1611 1612 #ifdef CONFIG_MOVABLE_NODE 1613 /* 1614 * When CONFIG_MOVABLE_NODE, we permit offlining of a node which doesn't have 1615 * normal memory. 1616 */ 1617 static bool can_offline_normal(struct zone *zone, unsigned long nr_pages) 1618 { 1619 return true; 1620 } 1621 #else /* CONFIG_MOVABLE_NODE */ 1622 /* ensure the node has NORMAL memory if it is still online */ 1623 static bool can_offline_normal(struct zone *zone, unsigned long nr_pages) 1624 { 1625 struct pglist_data *pgdat = zone->zone_pgdat; 1626 unsigned long present_pages = 0; 1627 enum zone_type zt; 1628 1629 for (zt = 0; zt <= ZONE_NORMAL; zt++) 1630 present_pages += pgdat->node_zones[zt].present_pages; 1631 1632 if (present_pages > nr_pages) 1633 return true; 1634 1635 present_pages = 0; 1636 for (; zt <= ZONE_MOVABLE; zt++) 1637 present_pages += pgdat->node_zones[zt].present_pages; 1638 1639 /* 1640 * we can't offline the last normal memory until all 1641 * higher memory is offlined. 1642 */ 1643 return present_pages == 0; 1644 } 1645 #endif /* CONFIG_MOVABLE_NODE */ 1646 1647 static int __init cmdline_parse_movable_node(char *p) 1648 { 1649 #ifdef CONFIG_MOVABLE_NODE 1650 /* 1651 * Memory used by the kernel cannot be hot-removed because Linux 1652 * cannot migrate the kernel pages. When memory hotplug is 1653 * enabled, we should prevent memblock from allocating memory 1654 * for the kernel. 1655 * 1656 * ACPI SRAT records all hotpluggable memory ranges. But before 1657 * SRAT is parsed, we don't know about it. 1658 * 1659 * The kernel image is loaded into memory at very early time. We 1660 * cannot prevent this anyway. So on NUMA system, we set any 1661 * node the kernel resides in as un-hotpluggable. 1662 * 1663 * Since on modern servers, one node could have double-digit 1664 * gigabytes memory, we can assume the memory around the kernel 1665 * image is also un-hotpluggable. So before SRAT is parsed, just 1666 * allocate memory near the kernel image to try the best to keep 1667 * the kernel away from hotpluggable memory. 1668 */ 1669 memblock_set_bottom_up(true); 1670 movable_node_enabled = true; 1671 #else 1672 pr_warn("movable_node option not supported\n"); 1673 #endif 1674 return 0; 1675 } 1676 early_param("movable_node", cmdline_parse_movable_node); 1677 1678 /* check which state of node_states will be changed when offline memory */ 1679 static void node_states_check_changes_offline(unsigned long nr_pages, 1680 struct zone *zone, struct memory_notify *arg) 1681 { 1682 struct pglist_data *pgdat = zone->zone_pgdat; 1683 unsigned long present_pages = 0; 1684 enum zone_type zt, zone_last = ZONE_NORMAL; 1685 1686 /* 1687 * If we have HIGHMEM or movable node, node_states[N_NORMAL_MEMORY] 1688 * contains nodes which have zones of 0...ZONE_NORMAL, 1689 * set zone_last to ZONE_NORMAL. 1690 * 1691 * If we don't have HIGHMEM nor movable node, 1692 * node_states[N_NORMAL_MEMORY] contains nodes which have zones of 1693 * 0...ZONE_MOVABLE, set zone_last to ZONE_MOVABLE. 1694 */ 1695 if (N_MEMORY == N_NORMAL_MEMORY) 1696 zone_last = ZONE_MOVABLE; 1697 1698 /* 1699 * check whether node_states[N_NORMAL_MEMORY] will be changed. 1700 * If the memory to be offline is in a zone of 0...zone_last, 1701 * and it is the last present memory, 0...zone_last will 1702 * become empty after offline , thus we can determind we will 1703 * need to clear the node from node_states[N_NORMAL_MEMORY]. 1704 */ 1705 for (zt = 0; zt <= zone_last; zt++) 1706 present_pages += pgdat->node_zones[zt].present_pages; 1707 if (zone_idx(zone) <= zone_last && nr_pages >= present_pages) 1708 arg->status_change_nid_normal = zone_to_nid(zone); 1709 else 1710 arg->status_change_nid_normal = -1; 1711 1712 #ifdef CONFIG_HIGHMEM 1713 /* 1714 * If we have movable node, node_states[N_HIGH_MEMORY] 1715 * contains nodes which have zones of 0...ZONE_HIGHMEM, 1716 * set zone_last to ZONE_HIGHMEM. 1717 * 1718 * If we don't have movable node, node_states[N_NORMAL_MEMORY] 1719 * contains nodes which have zones of 0...ZONE_MOVABLE, 1720 * set zone_last to ZONE_MOVABLE. 1721 */ 1722 zone_last = ZONE_HIGHMEM; 1723 if (N_MEMORY == N_HIGH_MEMORY) 1724 zone_last = ZONE_MOVABLE; 1725 1726 for (; zt <= zone_last; zt++) 1727 present_pages += pgdat->node_zones[zt].present_pages; 1728 if (zone_idx(zone) <= zone_last && nr_pages >= present_pages) 1729 arg->status_change_nid_high = zone_to_nid(zone); 1730 else 1731 arg->status_change_nid_high = -1; 1732 #else 1733 arg->status_change_nid_high = arg->status_change_nid_normal; 1734 #endif 1735 1736 /* 1737 * node_states[N_HIGH_MEMORY] contains nodes which have 0...ZONE_MOVABLE 1738 */ 1739 zone_last = ZONE_MOVABLE; 1740 1741 /* 1742 * check whether node_states[N_HIGH_MEMORY] will be changed 1743 * If we try to offline the last present @nr_pages from the node, 1744 * we can determind we will need to clear the node from 1745 * node_states[N_HIGH_MEMORY]. 1746 */ 1747 for (; zt <= zone_last; zt++) 1748 present_pages += pgdat->node_zones[zt].present_pages; 1749 if (nr_pages >= present_pages) 1750 arg->status_change_nid = zone_to_nid(zone); 1751 else 1752 arg->status_change_nid = -1; 1753 } 1754 1755 static void node_states_clear_node(int node, struct memory_notify *arg) 1756 { 1757 if (arg->status_change_nid_normal >= 0) 1758 node_clear_state(node, N_NORMAL_MEMORY); 1759 1760 if ((N_MEMORY != N_NORMAL_MEMORY) && 1761 (arg->status_change_nid_high >= 0)) 1762 node_clear_state(node, N_HIGH_MEMORY); 1763 1764 if ((N_MEMORY != N_HIGH_MEMORY) && 1765 (arg->status_change_nid >= 0)) 1766 node_clear_state(node, N_MEMORY); 1767 } 1768 1769 static int __ref __offline_pages(unsigned long start_pfn, 1770 unsigned long end_pfn, unsigned long timeout) 1771 { 1772 unsigned long pfn, nr_pages, expire; 1773 long offlined_pages; 1774 int ret, drain, retry_max, node; 1775 unsigned long flags; 1776 struct zone *zone; 1777 struct memory_notify arg; 1778 1779 /* at least, alignment against pageblock is necessary */ 1780 if (!IS_ALIGNED(start_pfn, pageblock_nr_pages)) 1781 return -EINVAL; 1782 if (!IS_ALIGNED(end_pfn, pageblock_nr_pages)) 1783 return -EINVAL; 1784 /* This makes hotplug much easier...and readable. 1785 we assume this for now. .*/ 1786 if (!test_pages_in_a_zone(start_pfn, end_pfn)) 1787 return -EINVAL; 1788 1789 zone = page_zone(pfn_to_page(start_pfn)); 1790 node = zone_to_nid(zone); 1791 nr_pages = end_pfn - start_pfn; 1792 1793 if (zone_idx(zone) <= ZONE_NORMAL && !can_offline_normal(zone, nr_pages)) 1794 return -EINVAL; 1795 1796 /* set above range as isolated */ 1797 ret = start_isolate_page_range(start_pfn, end_pfn, 1798 MIGRATE_MOVABLE, true); 1799 if (ret) 1800 return ret; 1801 1802 arg.start_pfn = start_pfn; 1803 arg.nr_pages = nr_pages; 1804 node_states_check_changes_offline(nr_pages, zone, &arg); 1805 1806 ret = memory_notify(MEM_GOING_OFFLINE, &arg); 1807 ret = notifier_to_errno(ret); 1808 if (ret) 1809 goto failed_removal; 1810 1811 pfn = start_pfn; 1812 expire = jiffies + timeout; 1813 drain = 0; 1814 retry_max = 5; 1815 repeat: 1816 /* start memory hot removal */ 1817 ret = -EAGAIN; 1818 if (time_after(jiffies, expire)) 1819 goto failed_removal; 1820 ret = -EINTR; 1821 if (signal_pending(current)) 1822 goto failed_removal; 1823 ret = 0; 1824 if (drain) { 1825 lru_add_drain_all(); 1826 cond_resched(); 1827 drain_all_pages(zone); 1828 } 1829 1830 pfn = scan_movable_pages(start_pfn, end_pfn); 1831 if (pfn) { /* We have movable pages */ 1832 ret = do_migrate_range(pfn, end_pfn); 1833 if (!ret) { 1834 drain = 1; 1835 goto repeat; 1836 } else { 1837 if (ret < 0) 1838 if (--retry_max == 0) 1839 goto failed_removal; 1840 yield(); 1841 drain = 1; 1842 goto repeat; 1843 } 1844 } 1845 /* drain all zone's lru pagevec, this is asynchronous... */ 1846 lru_add_drain_all(); 1847 yield(); 1848 /* drain pcp pages, this is synchronous. */ 1849 drain_all_pages(zone); 1850 /* 1851 * dissolve free hugepages in the memory block before doing offlining 1852 * actually in order to make hugetlbfs's object counting consistent. 1853 */ 1854 dissolve_free_huge_pages(start_pfn, end_pfn); 1855 /* check again */ 1856 offlined_pages = check_pages_isolated(start_pfn, end_pfn); 1857 if (offlined_pages < 0) { 1858 ret = -EBUSY; 1859 goto failed_removal; 1860 } 1861 pr_info("Offlined Pages %ld\n", offlined_pages); 1862 /* Ok, all of our target is isolated. 1863 We cannot do rollback at this point. */ 1864 offline_isolated_pages(start_pfn, end_pfn); 1865 /* reset pagetype flags and makes migrate type to be MOVABLE */ 1866 undo_isolate_page_range(start_pfn, end_pfn, MIGRATE_MOVABLE); 1867 /* removal success */ 1868 adjust_managed_page_count(pfn_to_page(start_pfn), -offlined_pages); 1869 zone->present_pages -= offlined_pages; 1870 1871 pgdat_resize_lock(zone->zone_pgdat, &flags); 1872 zone->zone_pgdat->node_present_pages -= offlined_pages; 1873 pgdat_resize_unlock(zone->zone_pgdat, &flags); 1874 1875 init_per_zone_wmark_min(); 1876 1877 if (!populated_zone(zone)) { 1878 zone_pcp_reset(zone); 1879 mutex_lock(&zonelists_mutex); 1880 build_all_zonelists(NULL, NULL); 1881 mutex_unlock(&zonelists_mutex); 1882 } else 1883 zone_pcp_update(zone); 1884 1885 node_states_clear_node(node, &arg); 1886 if (arg.status_change_nid >= 0) { 1887 kswapd_stop(node); 1888 kcompactd_stop(node); 1889 } 1890 1891 vm_total_pages = nr_free_pagecache_pages(); 1892 writeback_set_ratelimit(); 1893 1894 memory_notify(MEM_OFFLINE, &arg); 1895 return 0; 1896 1897 failed_removal: 1898 pr_debug("memory offlining [mem %#010llx-%#010llx] failed\n", 1899 (unsigned long long) start_pfn << PAGE_SHIFT, 1900 ((unsigned long long) end_pfn << PAGE_SHIFT) - 1); 1901 memory_notify(MEM_CANCEL_OFFLINE, &arg); 1902 /* pushback to free area */ 1903 undo_isolate_page_range(start_pfn, end_pfn, MIGRATE_MOVABLE); 1904 return ret; 1905 } 1906 1907 /* Must be protected by mem_hotplug_begin() */ 1908 int offline_pages(unsigned long start_pfn, unsigned long nr_pages) 1909 { 1910 return __offline_pages(start_pfn, start_pfn + nr_pages, 120 * HZ); 1911 } 1912 #endif /* CONFIG_MEMORY_HOTREMOVE */ 1913 1914 /** 1915 * walk_memory_range - walks through all mem sections in [start_pfn, end_pfn) 1916 * @start_pfn: start pfn of the memory range 1917 * @end_pfn: end pfn of the memory range 1918 * @arg: argument passed to func 1919 * @func: callback for each memory section walked 1920 * 1921 * This function walks through all present mem sections in range 1922 * [start_pfn, end_pfn) and call func on each mem section. 1923 * 1924 * Returns the return value of func. 1925 */ 1926 int walk_memory_range(unsigned long start_pfn, unsigned long end_pfn, 1927 void *arg, int (*func)(struct memory_block *, void *)) 1928 { 1929 struct memory_block *mem = NULL; 1930 struct mem_section *section; 1931 unsigned long pfn, section_nr; 1932 int ret; 1933 1934 for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) { 1935 section_nr = pfn_to_section_nr(pfn); 1936 if (!present_section_nr(section_nr)) 1937 continue; 1938 1939 section = __nr_to_section(section_nr); 1940 /* same memblock? */ 1941 if (mem) 1942 if ((section_nr >= mem->start_section_nr) && 1943 (section_nr <= mem->end_section_nr)) 1944 continue; 1945 1946 mem = find_memory_block_hinted(section, mem); 1947 if (!mem) 1948 continue; 1949 1950 ret = func(mem, arg); 1951 if (ret) { 1952 kobject_put(&mem->dev.kobj); 1953 return ret; 1954 } 1955 } 1956 1957 if (mem) 1958 kobject_put(&mem->dev.kobj); 1959 1960 return 0; 1961 } 1962 1963 #ifdef CONFIG_MEMORY_HOTREMOVE 1964 static int check_memblock_offlined_cb(struct memory_block *mem, void *arg) 1965 { 1966 int ret = !is_memblock_offlined(mem); 1967 1968 if (unlikely(ret)) { 1969 phys_addr_t beginpa, endpa; 1970 1971 beginpa = PFN_PHYS(section_nr_to_pfn(mem->start_section_nr)); 1972 endpa = PFN_PHYS(section_nr_to_pfn(mem->end_section_nr + 1))-1; 1973 pr_warn("removing memory fails, because memory [%pa-%pa] is onlined\n", 1974 &beginpa, &endpa); 1975 } 1976 1977 return ret; 1978 } 1979 1980 static int check_cpu_on_node(pg_data_t *pgdat) 1981 { 1982 int cpu; 1983 1984 for_each_present_cpu(cpu) { 1985 if (cpu_to_node(cpu) == pgdat->node_id) 1986 /* 1987 * the cpu on this node isn't removed, and we can't 1988 * offline this node. 1989 */ 1990 return -EBUSY; 1991 } 1992 1993 return 0; 1994 } 1995 1996 static void unmap_cpu_on_node(pg_data_t *pgdat) 1997 { 1998 #ifdef CONFIG_ACPI_NUMA 1999 int cpu; 2000 2001 for_each_possible_cpu(cpu) 2002 if (cpu_to_node(cpu) == pgdat->node_id) 2003 numa_clear_node(cpu); 2004 #endif 2005 } 2006 2007 static int check_and_unmap_cpu_on_node(pg_data_t *pgdat) 2008 { 2009 int ret; 2010 2011 ret = check_cpu_on_node(pgdat); 2012 if (ret) 2013 return ret; 2014 2015 /* 2016 * the node will be offlined when we come here, so we can clear 2017 * the cpu_to_node() now. 2018 */ 2019 2020 unmap_cpu_on_node(pgdat); 2021 return 0; 2022 } 2023 2024 /** 2025 * try_offline_node 2026 * 2027 * Offline a node if all memory sections and cpus of the node are removed. 2028 * 2029 * NOTE: The caller must call lock_device_hotplug() to serialize hotplug 2030 * and online/offline operations before this call. 2031 */ 2032 void try_offline_node(int nid) 2033 { 2034 pg_data_t *pgdat = NODE_DATA(nid); 2035 unsigned long start_pfn = pgdat->node_start_pfn; 2036 unsigned long end_pfn = start_pfn + pgdat->node_spanned_pages; 2037 unsigned long pfn; 2038 int i; 2039 2040 for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) { 2041 unsigned long section_nr = pfn_to_section_nr(pfn); 2042 2043 if (!present_section_nr(section_nr)) 2044 continue; 2045 2046 if (pfn_to_nid(pfn) != nid) 2047 continue; 2048 2049 /* 2050 * some memory sections of this node are not removed, and we 2051 * can't offline node now. 2052 */ 2053 return; 2054 } 2055 2056 if (check_and_unmap_cpu_on_node(pgdat)) 2057 return; 2058 2059 /* 2060 * all memory/cpu of this node are removed, we can offline this 2061 * node now. 2062 */ 2063 node_set_offline(nid); 2064 unregister_one_node(nid); 2065 2066 /* free waittable in each zone */ 2067 for (i = 0; i < MAX_NR_ZONES; i++) { 2068 struct zone *zone = pgdat->node_zones + i; 2069 2070 /* 2071 * wait_table may be allocated from boot memory, 2072 * here only free if it's allocated by vmalloc. 2073 */ 2074 if (is_vmalloc_addr(zone->wait_table)) { 2075 vfree(zone->wait_table); 2076 zone->wait_table = NULL; 2077 } 2078 } 2079 } 2080 EXPORT_SYMBOL(try_offline_node); 2081 2082 /** 2083 * remove_memory 2084 * 2085 * NOTE: The caller must call lock_device_hotplug() to serialize hotplug 2086 * and online/offline operations before this call, as required by 2087 * try_offline_node(). 2088 */ 2089 void __ref remove_memory(int nid, u64 start, u64 size) 2090 { 2091 int ret; 2092 2093 BUG_ON(check_hotplug_memory_range(start, size)); 2094 2095 mem_hotplug_begin(); 2096 2097 /* 2098 * All memory blocks must be offlined before removing memory. Check 2099 * whether all memory blocks in question are offline and trigger a BUG() 2100 * if this is not the case. 2101 */ 2102 ret = walk_memory_range(PFN_DOWN(start), PFN_UP(start + size - 1), NULL, 2103 check_memblock_offlined_cb); 2104 if (ret) 2105 BUG(); 2106 2107 /* remove memmap entry */ 2108 firmware_map_remove(start, start + size, "System RAM"); 2109 memblock_free(start, size); 2110 memblock_remove(start, size); 2111 2112 arch_remove_memory(start, size); 2113 2114 try_offline_node(nid); 2115 2116 mem_hotplug_done(); 2117 } 2118 EXPORT_SYMBOL_GPL(remove_memory); 2119 #endif /* CONFIG_MEMORY_HOTREMOVE */ 2120