1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * sparse memory mappings. 4 */ 5 #include <linux/mm.h> 6 #include <linux/slab.h> 7 #include <linux/mmzone.h> 8 #include <linux/memblock.h> 9 #include <linux/compiler.h> 10 #include <linux/highmem.h> 11 #include <linux/export.h> 12 #include <linux/spinlock.h> 13 #include <linux/vmalloc.h> 14 #include <linux/swap.h> 15 #include <linux/swapops.h> 16 #include <linux/bootmem_info.h> 17 #include <linux/vmstat.h> 18 #include "internal.h" 19 #include <asm/dma.h> 20 21 /* 22 * Permanent SPARSEMEM data: 23 * 24 * 1) mem_section - memory sections, mem_map's for valid memory 25 */ 26 #ifdef CONFIG_SPARSEMEM_EXTREME 27 struct mem_section **mem_section; 28 #else 29 struct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT] 30 ____cacheline_internodealigned_in_smp; 31 #endif 32 EXPORT_SYMBOL(mem_section); 33 34 #ifdef NODE_NOT_IN_PAGE_FLAGS 35 /* 36 * If we did not store the node number in the page then we have to 37 * do a lookup in the section_to_node_table in order to find which 38 * node the page belongs to. 39 */ 40 #if MAX_NUMNODES <= 256 41 static u8 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned; 42 #else 43 static u16 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned; 44 #endif 45 46 int page_to_nid(const struct page *page) 47 { 48 return section_to_node_table[page_to_section(page)]; 49 } 50 EXPORT_SYMBOL(page_to_nid); 51 52 static void set_section_nid(unsigned long section_nr, int nid) 53 { 54 section_to_node_table[section_nr] = nid; 55 } 56 #else /* !NODE_NOT_IN_PAGE_FLAGS */ 57 static inline void set_section_nid(unsigned long section_nr, int nid) 58 { 59 } 60 #endif 61 62 #ifdef CONFIG_SPARSEMEM_EXTREME 63 static noinline struct mem_section __ref *sparse_index_alloc(int nid) 64 { 65 struct mem_section *section = NULL; 66 unsigned long array_size = SECTIONS_PER_ROOT * 67 sizeof(struct mem_section); 68 69 if (slab_is_available()) { 70 section = kzalloc_node(array_size, GFP_KERNEL, nid); 71 } else { 72 section = memblock_alloc_node(array_size, SMP_CACHE_BYTES, 73 nid); 74 if (!section) 75 panic("%s: Failed to allocate %lu bytes nid=%d\n", 76 __func__, array_size, nid); 77 } 78 79 return section; 80 } 81 82 static int __meminit sparse_index_init(unsigned long section_nr, int nid) 83 { 84 unsigned long root = SECTION_NR_TO_ROOT(section_nr); 85 struct mem_section *section; 86 87 /* 88 * An existing section is possible in the sub-section hotplug 89 * case. First hot-add instantiates, follow-on hot-add reuses 90 * the existing section. 91 * 92 * The mem_hotplug_lock resolves the apparent race below. 93 */ 94 if (mem_section[root]) 95 return 0; 96 97 section = sparse_index_alloc(nid); 98 if (!section) 99 return -ENOMEM; 100 101 mem_section[root] = section; 102 103 return 0; 104 } 105 #else /* !SPARSEMEM_EXTREME */ 106 static inline int sparse_index_init(unsigned long section_nr, int nid) 107 { 108 return 0; 109 } 110 #endif 111 112 /* 113 * During early boot, before section_mem_map is used for an actual 114 * mem_map, we use section_mem_map to store the section's NUMA 115 * node. This keeps us from having to use another data structure. The 116 * node information is cleared just before we store the real mem_map. 117 */ 118 static inline unsigned long sparse_encode_early_nid(int nid) 119 { 120 return ((unsigned long)nid << SECTION_NID_SHIFT); 121 } 122 123 static inline int sparse_early_nid(struct mem_section *section) 124 { 125 return (section->section_mem_map >> SECTION_NID_SHIFT); 126 } 127 128 /* Validate the physical addressing limitations of the model */ 129 static void __meminit mminit_validate_memmodel_limits(unsigned long *start_pfn, 130 unsigned long *end_pfn) 131 { 132 unsigned long max_sparsemem_pfn = (PHYSMEM_END + 1) >> PAGE_SHIFT; 133 134 /* 135 * Sanity checks - do not allow an architecture to pass 136 * in larger pfns than the maximum scope of sparsemem: 137 */ 138 if (*start_pfn > max_sparsemem_pfn) { 139 mminit_dprintk(MMINIT_WARNING, "pfnvalidation", 140 "Start of range %lu -> %lu exceeds SPARSEMEM max %lu\n", 141 *start_pfn, *end_pfn, max_sparsemem_pfn); 142 WARN_ON_ONCE(1); 143 *start_pfn = max_sparsemem_pfn; 144 *end_pfn = max_sparsemem_pfn; 145 } else if (*end_pfn > max_sparsemem_pfn) { 146 mminit_dprintk(MMINIT_WARNING, "pfnvalidation", 147 "End of range %lu -> %lu exceeds SPARSEMEM max %lu\n", 148 *start_pfn, *end_pfn, max_sparsemem_pfn); 149 WARN_ON_ONCE(1); 150 *end_pfn = max_sparsemem_pfn; 151 } 152 } 153 154 /* 155 * There are a number of times that we loop over NR_MEM_SECTIONS, 156 * looking for section_present() on each. But, when we have very 157 * large physical address spaces, NR_MEM_SECTIONS can also be 158 * very large which makes the loops quite long. 159 * 160 * Keeping track of this gives us an easy way to break out of 161 * those loops early. 162 */ 163 unsigned long __highest_present_section_nr; 164 static void __section_mark_present(struct mem_section *ms, 165 unsigned long section_nr) 166 { 167 if (section_nr > __highest_present_section_nr) 168 __highest_present_section_nr = section_nr; 169 170 ms->section_mem_map |= SECTION_MARKED_PRESENT; 171 } 172 173 #define for_each_present_section_nr(start, section_nr) \ 174 for (section_nr = next_present_section_nr(start-1); \ 175 section_nr != -1; \ 176 section_nr = next_present_section_nr(section_nr)) 177 178 static inline unsigned long first_present_section_nr(void) 179 { 180 return next_present_section_nr(-1); 181 } 182 183 #ifdef CONFIG_SPARSEMEM_VMEMMAP 184 static void subsection_mask_set(unsigned long *map, unsigned long pfn, 185 unsigned long nr_pages) 186 { 187 int idx = subsection_map_index(pfn); 188 int end = subsection_map_index(pfn + nr_pages - 1); 189 190 bitmap_set(map, idx, end - idx + 1); 191 } 192 193 void __init subsection_map_init(unsigned long pfn, unsigned long nr_pages) 194 { 195 int end_sec_nr = pfn_to_section_nr(pfn + nr_pages - 1); 196 unsigned long nr, start_sec_nr = pfn_to_section_nr(pfn); 197 198 for (nr = start_sec_nr; nr <= end_sec_nr; nr++) { 199 struct mem_section *ms; 200 unsigned long pfns; 201 202 pfns = min(nr_pages, PAGES_PER_SECTION 203 - (pfn & ~PAGE_SECTION_MASK)); 204 ms = __nr_to_section(nr); 205 subsection_mask_set(ms->usage->subsection_map, pfn, pfns); 206 207 pr_debug("%s: sec: %lu pfns: %lu set(%d, %d)\n", __func__, nr, 208 pfns, subsection_map_index(pfn), 209 subsection_map_index(pfn + pfns - 1)); 210 211 pfn += pfns; 212 nr_pages -= pfns; 213 } 214 } 215 #else 216 void __init subsection_map_init(unsigned long pfn, unsigned long nr_pages) 217 { 218 } 219 #endif 220 221 /* Record a memory area against a node. */ 222 static void __init memory_present(int nid, unsigned long start, unsigned long end) 223 { 224 unsigned long pfn; 225 226 start &= PAGE_SECTION_MASK; 227 mminit_validate_memmodel_limits(&start, &end); 228 for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) { 229 unsigned long section_nr = pfn_to_section_nr(pfn); 230 struct mem_section *ms; 231 232 sparse_index_init(section_nr, nid); 233 set_section_nid(section_nr, nid); 234 235 ms = __nr_to_section(section_nr); 236 if (!ms->section_mem_map) { 237 ms->section_mem_map = sparse_encode_early_nid(nid) | 238 SECTION_IS_ONLINE; 239 __section_mark_present(ms, section_nr); 240 } 241 } 242 } 243 244 /* 245 * Mark all memblocks as present using memory_present(). 246 * This is a convenience function that is useful to mark all of the systems 247 * memory as present during initialization. 248 */ 249 static void __init memblocks_present(void) 250 { 251 unsigned long start, end; 252 int i, nid; 253 254 #ifdef CONFIG_SPARSEMEM_EXTREME 255 if (unlikely(!mem_section)) { 256 unsigned long size, align; 257 258 size = sizeof(struct mem_section *) * NR_SECTION_ROOTS; 259 align = 1 << (INTERNODE_CACHE_SHIFT); 260 mem_section = memblock_alloc(size, align); 261 if (!mem_section) 262 panic("%s: Failed to allocate %lu bytes align=0x%lx\n", 263 __func__, size, align); 264 } 265 #endif 266 267 for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) 268 memory_present(nid, start, end); 269 } 270 271 /* 272 * Subtle, we encode the real pfn into the mem_map such that 273 * the identity pfn - section_mem_map will return the actual 274 * physical page frame number. 275 */ 276 static unsigned long sparse_encode_mem_map(struct page *mem_map, unsigned long pnum) 277 { 278 unsigned long coded_mem_map = 279 (unsigned long)(mem_map - (section_nr_to_pfn(pnum))); 280 BUILD_BUG_ON(SECTION_MAP_LAST_BIT > PFN_SECTION_SHIFT); 281 BUG_ON(coded_mem_map & ~SECTION_MAP_MASK); 282 return coded_mem_map; 283 } 284 285 #ifdef CONFIG_MEMORY_HOTPLUG 286 /* 287 * Decode mem_map from the coded memmap 288 */ 289 struct page *sparse_decode_mem_map(unsigned long coded_mem_map, unsigned long pnum) 290 { 291 /* mask off the extra low bits of information */ 292 coded_mem_map &= SECTION_MAP_MASK; 293 return ((struct page *)coded_mem_map) + section_nr_to_pfn(pnum); 294 } 295 #endif /* CONFIG_MEMORY_HOTPLUG */ 296 297 static void __meminit sparse_init_one_section(struct mem_section *ms, 298 unsigned long pnum, struct page *mem_map, 299 struct mem_section_usage *usage, unsigned long flags) 300 { 301 ms->section_mem_map &= ~SECTION_MAP_MASK; 302 ms->section_mem_map |= sparse_encode_mem_map(mem_map, pnum) 303 | SECTION_HAS_MEM_MAP | flags; 304 ms->usage = usage; 305 } 306 307 static unsigned long usemap_size(void) 308 { 309 return BITS_TO_LONGS(SECTION_BLOCKFLAGS_BITS) * sizeof(unsigned long); 310 } 311 312 size_t mem_section_usage_size(void) 313 { 314 return sizeof(struct mem_section_usage) + usemap_size(); 315 } 316 317 #ifdef CONFIG_MEMORY_HOTREMOVE 318 static inline phys_addr_t pgdat_to_phys(struct pglist_data *pgdat) 319 { 320 #ifndef CONFIG_NUMA 321 VM_BUG_ON(pgdat != &contig_page_data); 322 return __pa_symbol(&contig_page_data); 323 #else 324 return __pa(pgdat); 325 #endif 326 } 327 328 static struct mem_section_usage * __init 329 sparse_early_usemaps_alloc_pgdat_section(struct pglist_data *pgdat, 330 unsigned long size) 331 { 332 struct mem_section_usage *usage; 333 unsigned long goal, limit; 334 int nid; 335 /* 336 * A page may contain usemaps for other sections preventing the 337 * page being freed and making a section unremovable while 338 * other sections referencing the usemap remain active. Similarly, 339 * a pgdat can prevent a section being removed. If section A 340 * contains a pgdat and section B contains the usemap, both 341 * sections become inter-dependent. This allocates usemaps 342 * from the same section as the pgdat where possible to avoid 343 * this problem. 344 */ 345 goal = pgdat_to_phys(pgdat) & (PAGE_SECTION_MASK << PAGE_SHIFT); 346 limit = goal + (1UL << PA_SECTION_SHIFT); 347 nid = early_pfn_to_nid(goal >> PAGE_SHIFT); 348 again: 349 usage = memblock_alloc_try_nid(size, SMP_CACHE_BYTES, goal, limit, nid); 350 if (!usage && limit) { 351 limit = MEMBLOCK_ALLOC_ACCESSIBLE; 352 goto again; 353 } 354 return usage; 355 } 356 357 static void __init check_usemap_section_nr(int nid, 358 struct mem_section_usage *usage) 359 { 360 unsigned long usemap_snr, pgdat_snr; 361 static unsigned long old_usemap_snr; 362 static unsigned long old_pgdat_snr; 363 struct pglist_data *pgdat = NODE_DATA(nid); 364 int usemap_nid; 365 366 /* First call */ 367 if (!old_usemap_snr) { 368 old_usemap_snr = NR_MEM_SECTIONS; 369 old_pgdat_snr = NR_MEM_SECTIONS; 370 } 371 372 usemap_snr = pfn_to_section_nr(__pa(usage) >> PAGE_SHIFT); 373 pgdat_snr = pfn_to_section_nr(pgdat_to_phys(pgdat) >> PAGE_SHIFT); 374 if (usemap_snr == pgdat_snr) 375 return; 376 377 if (old_usemap_snr == usemap_snr && old_pgdat_snr == pgdat_snr) 378 /* skip redundant message */ 379 return; 380 381 old_usemap_snr = usemap_snr; 382 old_pgdat_snr = pgdat_snr; 383 384 usemap_nid = sparse_early_nid(__nr_to_section(usemap_snr)); 385 if (usemap_nid != nid) { 386 pr_info("node %d must be removed before remove section %ld\n", 387 nid, usemap_snr); 388 return; 389 } 390 /* 391 * There is a circular dependency. 392 * Some platforms allow un-removable section because they will just 393 * gather other removable sections for dynamic partitioning. 394 * Just notify un-removable section's number here. 395 */ 396 pr_info("Section %ld and %ld (node %d) have a circular dependency on usemap and pgdat allocations\n", 397 usemap_snr, pgdat_snr, nid); 398 } 399 #else 400 static struct mem_section_usage * __init 401 sparse_early_usemaps_alloc_pgdat_section(struct pglist_data *pgdat, 402 unsigned long size) 403 { 404 return memblock_alloc_node(size, SMP_CACHE_BYTES, pgdat->node_id); 405 } 406 407 static void __init check_usemap_section_nr(int nid, 408 struct mem_section_usage *usage) 409 { 410 } 411 #endif /* CONFIG_MEMORY_HOTREMOVE */ 412 413 #ifdef CONFIG_SPARSEMEM_VMEMMAP 414 static unsigned long __init section_map_size(void) 415 { 416 return ALIGN(sizeof(struct page) * PAGES_PER_SECTION, PMD_SIZE); 417 } 418 419 #else 420 static unsigned long __init section_map_size(void) 421 { 422 return PAGE_ALIGN(sizeof(struct page) * PAGES_PER_SECTION); 423 } 424 425 struct page __init *__populate_section_memmap(unsigned long pfn, 426 unsigned long nr_pages, int nid, struct vmem_altmap *altmap, 427 struct dev_pagemap *pgmap) 428 { 429 unsigned long size = section_map_size(); 430 struct page *map = sparse_buffer_alloc(size); 431 phys_addr_t addr = __pa(MAX_DMA_ADDRESS); 432 433 if (map) 434 return map; 435 436 map = memmap_alloc(size, size, addr, nid, false); 437 if (!map) 438 panic("%s: Failed to allocate %lu bytes align=0x%lx nid=%d from=%pa\n", 439 __func__, size, PAGE_SIZE, nid, &addr); 440 441 return map; 442 } 443 #endif /* !CONFIG_SPARSEMEM_VMEMMAP */ 444 445 static void *sparsemap_buf __meminitdata; 446 static void *sparsemap_buf_end __meminitdata; 447 448 static inline void __meminit sparse_buffer_free(unsigned long size) 449 { 450 WARN_ON(!sparsemap_buf || size == 0); 451 memblock_free(sparsemap_buf, size); 452 } 453 454 static void __init sparse_buffer_init(unsigned long size, int nid) 455 { 456 phys_addr_t addr = __pa(MAX_DMA_ADDRESS); 457 WARN_ON(sparsemap_buf); /* forgot to call sparse_buffer_fini()? */ 458 /* 459 * Pre-allocated buffer is mainly used by __populate_section_memmap 460 * and we want it to be properly aligned to the section size - this is 461 * especially the case for VMEMMAP which maps memmap to PMDs 462 */ 463 sparsemap_buf = memmap_alloc(size, section_map_size(), addr, nid, true); 464 sparsemap_buf_end = sparsemap_buf + size; 465 #ifndef CONFIG_SPARSEMEM_VMEMMAP 466 memmap_boot_pages_add(DIV_ROUND_UP(size, PAGE_SIZE)); 467 #endif 468 } 469 470 static void __init sparse_buffer_fini(void) 471 { 472 unsigned long size = sparsemap_buf_end - sparsemap_buf; 473 474 if (sparsemap_buf && size > 0) 475 sparse_buffer_free(size); 476 sparsemap_buf = NULL; 477 } 478 479 void * __meminit sparse_buffer_alloc(unsigned long size) 480 { 481 void *ptr = NULL; 482 483 if (sparsemap_buf) { 484 ptr = (void *) roundup((unsigned long)sparsemap_buf, size); 485 if (ptr + size > sparsemap_buf_end) 486 ptr = NULL; 487 else { 488 /* Free redundant aligned space */ 489 if ((unsigned long)(ptr - sparsemap_buf) > 0) 490 sparse_buffer_free((unsigned long)(ptr - sparsemap_buf)); 491 sparsemap_buf = ptr + size; 492 } 493 } 494 return ptr; 495 } 496 497 void __weak __meminit vmemmap_populate_print_last(void) 498 { 499 } 500 501 /* 502 * Initialize sparse on a specific node. The node spans [pnum_begin, pnum_end) 503 * And number of present sections in this node is map_count. 504 */ 505 static void __init sparse_init_nid(int nid, unsigned long pnum_begin, 506 unsigned long pnum_end, 507 unsigned long map_count) 508 { 509 struct mem_section_usage *usage; 510 unsigned long pnum; 511 struct page *map; 512 513 usage = sparse_early_usemaps_alloc_pgdat_section(NODE_DATA(nid), 514 mem_section_usage_size() * map_count); 515 if (!usage) { 516 pr_err("%s: node[%d] usemap allocation failed", __func__, nid); 517 goto failed; 518 } 519 sparse_buffer_init(map_count * section_map_size(), nid); 520 for_each_present_section_nr(pnum_begin, pnum) { 521 unsigned long pfn = section_nr_to_pfn(pnum); 522 523 if (pnum >= pnum_end) 524 break; 525 526 map = __populate_section_memmap(pfn, PAGES_PER_SECTION, 527 nid, NULL, NULL); 528 if (!map) { 529 pr_err("%s: node[%d] memory map backing failed. Some memory will not be available.", 530 __func__, nid); 531 pnum_begin = pnum; 532 sparse_buffer_fini(); 533 goto failed; 534 } 535 check_usemap_section_nr(nid, usage); 536 sparse_init_one_section(__nr_to_section(pnum), pnum, map, usage, 537 SECTION_IS_EARLY); 538 usage = (void *) usage + mem_section_usage_size(); 539 } 540 sparse_buffer_fini(); 541 return; 542 failed: 543 /* We failed to allocate, mark all the following pnums as not present */ 544 for_each_present_section_nr(pnum_begin, pnum) { 545 struct mem_section *ms; 546 547 if (pnum >= pnum_end) 548 break; 549 ms = __nr_to_section(pnum); 550 ms->section_mem_map = 0; 551 } 552 } 553 554 /* 555 * Allocate the accumulated non-linear sections, allocate a mem_map 556 * for each and record the physical to section mapping. 557 */ 558 void __init sparse_init(void) 559 { 560 unsigned long pnum_end, pnum_begin, map_count = 1; 561 int nid_begin; 562 563 /* see include/linux/mmzone.h 'struct mem_section' definition */ 564 BUILD_BUG_ON(!is_power_of_2(sizeof(struct mem_section))); 565 memblocks_present(); 566 567 pnum_begin = first_present_section_nr(); 568 nid_begin = sparse_early_nid(__nr_to_section(pnum_begin)); 569 570 /* Setup pageblock_order for HUGETLB_PAGE_SIZE_VARIABLE */ 571 set_pageblock_order(); 572 573 for_each_present_section_nr(pnum_begin + 1, pnum_end) { 574 int nid = sparse_early_nid(__nr_to_section(pnum_end)); 575 576 if (nid == nid_begin) { 577 map_count++; 578 continue; 579 } 580 /* Init node with sections in range [pnum_begin, pnum_end) */ 581 sparse_init_nid(nid_begin, pnum_begin, pnum_end, map_count); 582 nid_begin = nid; 583 pnum_begin = pnum_end; 584 map_count = 1; 585 } 586 /* cover the last node */ 587 sparse_init_nid(nid_begin, pnum_begin, pnum_end, map_count); 588 vmemmap_populate_print_last(); 589 } 590 591 #ifdef CONFIG_MEMORY_HOTPLUG 592 593 /* Mark all memory sections within the pfn range as online */ 594 void online_mem_sections(unsigned long start_pfn, unsigned long end_pfn) 595 { 596 unsigned long pfn; 597 598 for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) { 599 unsigned long section_nr = pfn_to_section_nr(pfn); 600 struct mem_section *ms; 601 602 /* onlining code should never touch invalid ranges */ 603 if (WARN_ON(!valid_section_nr(section_nr))) 604 continue; 605 606 ms = __nr_to_section(section_nr); 607 ms->section_mem_map |= SECTION_IS_ONLINE; 608 } 609 } 610 611 /* Mark all memory sections within the pfn range as offline */ 612 void offline_mem_sections(unsigned long start_pfn, unsigned long end_pfn) 613 { 614 unsigned long pfn; 615 616 for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) { 617 unsigned long section_nr = pfn_to_section_nr(pfn); 618 struct mem_section *ms; 619 620 /* 621 * TODO this needs some double checking. Offlining code makes 622 * sure to check pfn_valid but those checks might be just bogus 623 */ 624 if (WARN_ON(!valid_section_nr(section_nr))) 625 continue; 626 627 ms = __nr_to_section(section_nr); 628 ms->section_mem_map &= ~SECTION_IS_ONLINE; 629 } 630 } 631 632 #ifdef CONFIG_SPARSEMEM_VMEMMAP 633 static struct page * __meminit populate_section_memmap(unsigned long pfn, 634 unsigned long nr_pages, int nid, struct vmem_altmap *altmap, 635 struct dev_pagemap *pgmap) 636 { 637 return __populate_section_memmap(pfn, nr_pages, nid, altmap, pgmap); 638 } 639 640 static void depopulate_section_memmap(unsigned long pfn, unsigned long nr_pages, 641 struct vmem_altmap *altmap) 642 { 643 unsigned long start = (unsigned long) pfn_to_page(pfn); 644 unsigned long end = start + nr_pages * sizeof(struct page); 645 646 memmap_pages_add(-1L * (DIV_ROUND_UP(end - start, PAGE_SIZE))); 647 vmemmap_free(start, end, altmap); 648 } 649 static void free_map_bootmem(struct page *memmap) 650 { 651 unsigned long start = (unsigned long)memmap; 652 unsigned long end = (unsigned long)(memmap + PAGES_PER_SECTION); 653 654 vmemmap_free(start, end, NULL); 655 } 656 657 static int clear_subsection_map(unsigned long pfn, unsigned long nr_pages) 658 { 659 DECLARE_BITMAP(map, SUBSECTIONS_PER_SECTION) = { 0 }; 660 DECLARE_BITMAP(tmp, SUBSECTIONS_PER_SECTION) = { 0 }; 661 struct mem_section *ms = __pfn_to_section(pfn); 662 unsigned long *subsection_map = ms->usage 663 ? &ms->usage->subsection_map[0] : NULL; 664 665 subsection_mask_set(map, pfn, nr_pages); 666 if (subsection_map) 667 bitmap_and(tmp, map, subsection_map, SUBSECTIONS_PER_SECTION); 668 669 if (WARN(!subsection_map || !bitmap_equal(tmp, map, SUBSECTIONS_PER_SECTION), 670 "section already deactivated (%#lx + %ld)\n", 671 pfn, nr_pages)) 672 return -EINVAL; 673 674 bitmap_xor(subsection_map, map, subsection_map, SUBSECTIONS_PER_SECTION); 675 return 0; 676 } 677 678 static bool is_subsection_map_empty(struct mem_section *ms) 679 { 680 return bitmap_empty(&ms->usage->subsection_map[0], 681 SUBSECTIONS_PER_SECTION); 682 } 683 684 static int fill_subsection_map(unsigned long pfn, unsigned long nr_pages) 685 { 686 struct mem_section *ms = __pfn_to_section(pfn); 687 DECLARE_BITMAP(map, SUBSECTIONS_PER_SECTION) = { 0 }; 688 unsigned long *subsection_map; 689 int rc = 0; 690 691 subsection_mask_set(map, pfn, nr_pages); 692 693 subsection_map = &ms->usage->subsection_map[0]; 694 695 if (bitmap_empty(map, SUBSECTIONS_PER_SECTION)) 696 rc = -EINVAL; 697 else if (bitmap_intersects(map, subsection_map, SUBSECTIONS_PER_SECTION)) 698 rc = -EEXIST; 699 else 700 bitmap_or(subsection_map, map, subsection_map, 701 SUBSECTIONS_PER_SECTION); 702 703 return rc; 704 } 705 #else 706 static struct page * __meminit populate_section_memmap(unsigned long pfn, 707 unsigned long nr_pages, int nid, struct vmem_altmap *altmap, 708 struct dev_pagemap *pgmap) 709 { 710 return kvmalloc_node(array_size(sizeof(struct page), 711 PAGES_PER_SECTION), GFP_KERNEL, nid); 712 } 713 714 static void depopulate_section_memmap(unsigned long pfn, unsigned long nr_pages, 715 struct vmem_altmap *altmap) 716 { 717 kvfree(pfn_to_page(pfn)); 718 } 719 720 static void free_map_bootmem(struct page *memmap) 721 { 722 unsigned long maps_section_nr, removing_section_nr, i; 723 unsigned long magic, nr_pages; 724 struct page *page = virt_to_page(memmap); 725 726 nr_pages = PAGE_ALIGN(PAGES_PER_SECTION * sizeof(struct page)) 727 >> PAGE_SHIFT; 728 729 for (i = 0; i < nr_pages; i++, page++) { 730 magic = page->index; 731 732 BUG_ON(magic == NODE_INFO); 733 734 maps_section_nr = pfn_to_section_nr(page_to_pfn(page)); 735 removing_section_nr = page_private(page); 736 737 /* 738 * When this function is called, the removing section is 739 * logical offlined state. This means all pages are isolated 740 * from page allocator. If removing section's memmap is placed 741 * on the same section, it must not be freed. 742 * If it is freed, page allocator may allocate it which will 743 * be removed physically soon. 744 */ 745 if (maps_section_nr != removing_section_nr) 746 put_page_bootmem(page); 747 } 748 } 749 750 static int clear_subsection_map(unsigned long pfn, unsigned long nr_pages) 751 { 752 return 0; 753 } 754 755 static bool is_subsection_map_empty(struct mem_section *ms) 756 { 757 return true; 758 } 759 760 static int fill_subsection_map(unsigned long pfn, unsigned long nr_pages) 761 { 762 return 0; 763 } 764 #endif /* CONFIG_SPARSEMEM_VMEMMAP */ 765 766 /* 767 * To deactivate a memory region, there are 3 cases to handle across 768 * two configurations (SPARSEMEM_VMEMMAP={y,n}): 769 * 770 * 1. deactivation of a partial hot-added section (only possible in 771 * the SPARSEMEM_VMEMMAP=y case). 772 * a) section was present at memory init. 773 * b) section was hot-added post memory init. 774 * 2. deactivation of a complete hot-added section. 775 * 3. deactivation of a complete section from memory init. 776 * 777 * For 1, when subsection_map does not empty we will not be freeing the 778 * usage map, but still need to free the vmemmap range. 779 * 780 * For 2 and 3, the SPARSEMEM_VMEMMAP={y,n} cases are unified 781 */ 782 static void section_deactivate(unsigned long pfn, unsigned long nr_pages, 783 struct vmem_altmap *altmap) 784 { 785 struct mem_section *ms = __pfn_to_section(pfn); 786 bool section_is_early = early_section(ms); 787 struct page *memmap = NULL; 788 bool empty; 789 790 if (clear_subsection_map(pfn, nr_pages)) 791 return; 792 793 empty = is_subsection_map_empty(ms); 794 if (empty) { 795 unsigned long section_nr = pfn_to_section_nr(pfn); 796 797 /* 798 * Mark the section invalid so that valid_section() 799 * return false. This prevents code from dereferencing 800 * ms->usage array. 801 */ 802 ms->section_mem_map &= ~SECTION_HAS_MEM_MAP; 803 804 /* 805 * When removing an early section, the usage map is kept (as the 806 * usage maps of other sections fall into the same page). It 807 * will be re-used when re-adding the section - which is then no 808 * longer an early section. If the usage map is PageReserved, it 809 * was allocated during boot. 810 */ 811 if (!PageReserved(virt_to_page(ms->usage))) { 812 kfree_rcu(ms->usage, rcu); 813 WRITE_ONCE(ms->usage, NULL); 814 } 815 memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr); 816 } 817 818 /* 819 * The memmap of early sections is always fully populated. See 820 * section_activate() and pfn_valid() . 821 */ 822 if (!section_is_early) 823 depopulate_section_memmap(pfn, nr_pages, altmap); 824 else if (memmap) 825 free_map_bootmem(memmap); 826 827 if (empty) 828 ms->section_mem_map = (unsigned long)NULL; 829 } 830 831 static struct page * __meminit section_activate(int nid, unsigned long pfn, 832 unsigned long nr_pages, struct vmem_altmap *altmap, 833 struct dev_pagemap *pgmap) 834 { 835 struct mem_section *ms = __pfn_to_section(pfn); 836 struct mem_section_usage *usage = NULL; 837 struct page *memmap; 838 int rc; 839 840 if (!ms->usage) { 841 usage = kzalloc(mem_section_usage_size(), GFP_KERNEL); 842 if (!usage) 843 return ERR_PTR(-ENOMEM); 844 ms->usage = usage; 845 } 846 847 rc = fill_subsection_map(pfn, nr_pages); 848 if (rc) { 849 if (usage) 850 ms->usage = NULL; 851 kfree(usage); 852 return ERR_PTR(rc); 853 } 854 855 /* 856 * The early init code does not consider partially populated 857 * initial sections, it simply assumes that memory will never be 858 * referenced. If we hot-add memory into such a section then we 859 * do not need to populate the memmap and can simply reuse what 860 * is already there. 861 */ 862 if (nr_pages < PAGES_PER_SECTION && early_section(ms)) 863 return pfn_to_page(pfn); 864 865 memmap = populate_section_memmap(pfn, nr_pages, nid, altmap, pgmap); 866 if (!memmap) { 867 section_deactivate(pfn, nr_pages, altmap); 868 return ERR_PTR(-ENOMEM); 869 } 870 871 return memmap; 872 } 873 874 /** 875 * sparse_add_section - add a memory section, or populate an existing one 876 * @nid: The node to add section on 877 * @start_pfn: start pfn of the memory range 878 * @nr_pages: number of pfns to add in the section 879 * @altmap: alternate pfns to allocate the memmap backing store 880 * @pgmap: alternate compound page geometry for devmap mappings 881 * 882 * This is only intended for hotplug. 883 * 884 * Note that only VMEMMAP supports sub-section aligned hotplug, 885 * the proper alignment and size are gated by check_pfn_span(). 886 * 887 * 888 * Return: 889 * * 0 - On success. 890 * * -EEXIST - Section has been present. 891 * * -ENOMEM - Out of memory. 892 */ 893 int __meminit sparse_add_section(int nid, unsigned long start_pfn, 894 unsigned long nr_pages, struct vmem_altmap *altmap, 895 struct dev_pagemap *pgmap) 896 { 897 unsigned long section_nr = pfn_to_section_nr(start_pfn); 898 struct mem_section *ms; 899 struct page *memmap; 900 int ret; 901 902 ret = sparse_index_init(section_nr, nid); 903 if (ret < 0) 904 return ret; 905 906 memmap = section_activate(nid, start_pfn, nr_pages, altmap, pgmap); 907 if (IS_ERR(memmap)) 908 return PTR_ERR(memmap); 909 910 /* 911 * Poison uninitialized struct pages in order to catch invalid flags 912 * combinations. 913 */ 914 if (!altmap || !altmap->inaccessible) 915 page_init_poison(memmap, sizeof(struct page) * nr_pages); 916 917 ms = __nr_to_section(section_nr); 918 set_section_nid(section_nr, nid); 919 __section_mark_present(ms, section_nr); 920 921 /* Align memmap to section boundary in the subsection case */ 922 if (section_nr_to_pfn(section_nr) != start_pfn) 923 memmap = pfn_to_page(section_nr_to_pfn(section_nr)); 924 sparse_init_one_section(ms, section_nr, memmap, ms->usage, 0); 925 926 return 0; 927 } 928 929 void sparse_remove_section(unsigned long pfn, unsigned long nr_pages, 930 struct vmem_altmap *altmap) 931 { 932 struct mem_section *ms = __pfn_to_section(pfn); 933 934 if (WARN_ON_ONCE(!valid_section(ms))) 935 return; 936 937 section_deactivate(pfn, nr_pages, altmap); 938 } 939 #endif /* CONFIG_MEMORY_HOTPLUG */ 940