1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * mm_init.c - Memory initialisation verification and debugging 4 * 5 * Copyright 2008 IBM Corporation, 2008 6 * Author Mel Gorman <mel@csn.ul.ie> 7 * 8 */ 9 #include <linux/kernel.h> 10 #include <linux/init.h> 11 #include <linux/kobject.h> 12 #include <linux/export.h> 13 #include <linux/memory.h> 14 #include <linux/notifier.h> 15 #include <linux/sched.h> 16 #include <linux/mman.h> 17 #include <linux/memblock.h> 18 #include <linux/page-isolation.h> 19 #include <linux/padata.h> 20 #include <linux/nmi.h> 21 #include <linux/buffer_head.h> 22 #include <linux/kmemleak.h> 23 #include <linux/kfence.h> 24 #include <linux/page_ext.h> 25 #include <linux/pti.h> 26 #include <linux/pgtable.h> 27 #include <linux/swap.h> 28 #include <linux/cma.h> 29 #include "internal.h" 30 #include "slab.h" 31 #include "shuffle.h" 32 33 #include <asm/setup.h> 34 35 #ifdef CONFIG_DEBUG_MEMORY_INIT 36 int __meminitdata mminit_loglevel; 37 38 /* The zonelists are simply reported, validation is manual. */ 39 void __init mminit_verify_zonelist(void) 40 { 41 int nid; 42 43 if (mminit_loglevel < MMINIT_VERIFY) 44 return; 45 46 for_each_online_node(nid) { 47 pg_data_t *pgdat = NODE_DATA(nid); 48 struct zone *zone; 49 struct zoneref *z; 50 struct zonelist *zonelist; 51 int i, listid, zoneid; 52 53 BUILD_BUG_ON(MAX_ZONELISTS > 2); 54 for (i = 0; i < MAX_ZONELISTS * MAX_NR_ZONES; i++) { 55 56 /* Identify the zone and nodelist */ 57 zoneid = i % MAX_NR_ZONES; 58 listid = i / MAX_NR_ZONES; 59 zonelist = &pgdat->node_zonelists[listid]; 60 zone = &pgdat->node_zones[zoneid]; 61 if (!populated_zone(zone)) 62 continue; 63 64 /* Print information about the zonelist */ 65 printk(KERN_DEBUG "mminit::zonelist %s %d:%s = ", 66 listid > 0 ? "thisnode" : "general", nid, 67 zone->name); 68 69 /* Iterate the zonelist */ 70 for_each_zone_zonelist(zone, z, zonelist, zoneid) 71 pr_cont("%d:%s ", zone_to_nid(zone), zone->name); 72 pr_cont("\n"); 73 } 74 } 75 } 76 77 void __init mminit_verify_pageflags_layout(void) 78 { 79 int shift, width; 80 unsigned long or_mask, add_mask; 81 82 shift = BITS_PER_LONG; 83 width = shift - SECTIONS_WIDTH - NODES_WIDTH - ZONES_WIDTH 84 - LAST_CPUPID_SHIFT - KASAN_TAG_WIDTH - LRU_GEN_WIDTH - LRU_REFS_WIDTH; 85 mminit_dprintk(MMINIT_TRACE, "pageflags_layout_widths", 86 "Section %d Node %d Zone %d Lastcpupid %d Kasantag %d Gen %d Tier %d Flags %d\n", 87 SECTIONS_WIDTH, 88 NODES_WIDTH, 89 ZONES_WIDTH, 90 LAST_CPUPID_WIDTH, 91 KASAN_TAG_WIDTH, 92 LRU_GEN_WIDTH, 93 LRU_REFS_WIDTH, 94 NR_PAGEFLAGS); 95 mminit_dprintk(MMINIT_TRACE, "pageflags_layout_shifts", 96 "Section %d Node %d Zone %d Lastcpupid %d Kasantag %d\n", 97 SECTIONS_SHIFT, 98 NODES_SHIFT, 99 ZONES_SHIFT, 100 LAST_CPUPID_SHIFT, 101 KASAN_TAG_WIDTH); 102 mminit_dprintk(MMINIT_TRACE, "pageflags_layout_pgshifts", 103 "Section %lu Node %lu Zone %lu Lastcpupid %lu Kasantag %lu\n", 104 (unsigned long)SECTIONS_PGSHIFT, 105 (unsigned long)NODES_PGSHIFT, 106 (unsigned long)ZONES_PGSHIFT, 107 (unsigned long)LAST_CPUPID_PGSHIFT, 108 (unsigned long)KASAN_TAG_PGSHIFT); 109 mminit_dprintk(MMINIT_TRACE, "pageflags_layout_nodezoneid", 110 "Node/Zone ID: %lu -> %lu\n", 111 (unsigned long)(ZONEID_PGOFF + ZONEID_SHIFT), 112 (unsigned long)ZONEID_PGOFF); 113 mminit_dprintk(MMINIT_TRACE, "pageflags_layout_usage", 114 "location: %d -> %d layout %d -> %d unused %d -> %d page-flags\n", 115 shift, width, width, NR_PAGEFLAGS, NR_PAGEFLAGS, 0); 116 #ifdef NODE_NOT_IN_PAGE_FLAGS 117 mminit_dprintk(MMINIT_TRACE, "pageflags_layout_nodeflags", 118 "Node not in page flags"); 119 #endif 120 #ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS 121 mminit_dprintk(MMINIT_TRACE, "pageflags_layout_nodeflags", 122 "Last cpupid not in page flags"); 123 #endif 124 125 if (SECTIONS_WIDTH) { 126 shift -= SECTIONS_WIDTH; 127 BUG_ON(shift != SECTIONS_PGSHIFT); 128 } 129 if (NODES_WIDTH) { 130 shift -= NODES_WIDTH; 131 BUG_ON(shift != NODES_PGSHIFT); 132 } 133 if (ZONES_WIDTH) { 134 shift -= ZONES_WIDTH; 135 BUG_ON(shift != ZONES_PGSHIFT); 136 } 137 138 /* Check for bitmask overlaps */ 139 or_mask = (ZONES_MASK << ZONES_PGSHIFT) | 140 (NODES_MASK << NODES_PGSHIFT) | 141 (SECTIONS_MASK << SECTIONS_PGSHIFT); 142 add_mask = (ZONES_MASK << ZONES_PGSHIFT) + 143 (NODES_MASK << NODES_PGSHIFT) + 144 (SECTIONS_MASK << SECTIONS_PGSHIFT); 145 BUG_ON(or_mask != add_mask); 146 } 147 148 static __init int set_mminit_loglevel(char *str) 149 { 150 get_option(&str, &mminit_loglevel); 151 return 0; 152 } 153 early_param("mminit_loglevel", set_mminit_loglevel); 154 #endif /* CONFIG_DEBUG_MEMORY_INIT */ 155 156 struct kobject *mm_kobj; 157 158 #ifdef CONFIG_SMP 159 s32 vm_committed_as_batch = 32; 160 161 void mm_compute_batch(int overcommit_policy) 162 { 163 u64 memsized_batch; 164 s32 nr = num_present_cpus(); 165 s32 batch = max_t(s32, nr*2, 32); 166 unsigned long ram_pages = totalram_pages(); 167 168 /* 169 * For policy OVERCOMMIT_NEVER, set batch size to 0.4% of 170 * (total memory/#cpus), and lift it to 25% for other policies 171 * to easy the possible lock contention for percpu_counter 172 * vm_committed_as, while the max limit is INT_MAX 173 */ 174 if (overcommit_policy == OVERCOMMIT_NEVER) 175 memsized_batch = min_t(u64, ram_pages/nr/256, INT_MAX); 176 else 177 memsized_batch = min_t(u64, ram_pages/nr/4, INT_MAX); 178 179 vm_committed_as_batch = max_t(s32, memsized_batch, batch); 180 } 181 182 static int __meminit mm_compute_batch_notifier(struct notifier_block *self, 183 unsigned long action, void *arg) 184 { 185 switch (action) { 186 case MEM_ONLINE: 187 case MEM_OFFLINE: 188 mm_compute_batch(sysctl_overcommit_memory); 189 break; 190 default: 191 break; 192 } 193 return NOTIFY_OK; 194 } 195 196 static int __init mm_compute_batch_init(void) 197 { 198 mm_compute_batch(sysctl_overcommit_memory); 199 hotplug_memory_notifier(mm_compute_batch_notifier, MM_COMPUTE_BATCH_PRI); 200 return 0; 201 } 202 203 __initcall(mm_compute_batch_init); 204 205 #endif 206 207 static int __init mm_sysfs_init(void) 208 { 209 mm_kobj = kobject_create_and_add("mm", kernel_kobj); 210 if (!mm_kobj) 211 return -ENOMEM; 212 213 return 0; 214 } 215 postcore_initcall(mm_sysfs_init); 216 217 static unsigned long arch_zone_lowest_possible_pfn[MAX_NR_ZONES] __initdata; 218 static unsigned long arch_zone_highest_possible_pfn[MAX_NR_ZONES] __initdata; 219 static unsigned long zone_movable_pfn[MAX_NUMNODES] __initdata; 220 221 static unsigned long required_kernelcore __initdata; 222 static unsigned long required_kernelcore_percent __initdata; 223 static unsigned long required_movablecore __initdata; 224 static unsigned long required_movablecore_percent __initdata; 225 226 static unsigned long nr_kernel_pages __initdata; 227 static unsigned long nr_all_pages __initdata; 228 static unsigned long dma_reserve __initdata; 229 230 static bool deferred_struct_pages __meminitdata; 231 232 static DEFINE_PER_CPU(struct per_cpu_nodestat, boot_nodestats); 233 234 static int __init cmdline_parse_core(char *p, unsigned long *core, 235 unsigned long *percent) 236 { 237 unsigned long long coremem; 238 char *endptr; 239 240 if (!p) 241 return -EINVAL; 242 243 /* Value may be a percentage of total memory, otherwise bytes */ 244 coremem = simple_strtoull(p, &endptr, 0); 245 if (*endptr == '%') { 246 /* Paranoid check for percent values greater than 100 */ 247 WARN_ON(coremem > 100); 248 249 *percent = coremem; 250 } else { 251 coremem = memparse(p, &p); 252 /* Paranoid check that UL is enough for the coremem value */ 253 WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX); 254 255 *core = coremem >> PAGE_SHIFT; 256 *percent = 0UL; 257 } 258 return 0; 259 } 260 261 bool mirrored_kernelcore __initdata_memblock; 262 263 /* 264 * kernelcore=size sets the amount of memory for use for allocations that 265 * cannot be reclaimed or migrated. 266 */ 267 static int __init cmdline_parse_kernelcore(char *p) 268 { 269 /* parse kernelcore=mirror */ 270 if (parse_option_str(p, "mirror")) { 271 mirrored_kernelcore = true; 272 return 0; 273 } 274 275 return cmdline_parse_core(p, &required_kernelcore, 276 &required_kernelcore_percent); 277 } 278 early_param("kernelcore", cmdline_parse_kernelcore); 279 280 /* 281 * movablecore=size sets the amount of memory for use for allocations that 282 * can be reclaimed or migrated. 283 */ 284 static int __init cmdline_parse_movablecore(char *p) 285 { 286 return cmdline_parse_core(p, &required_movablecore, 287 &required_movablecore_percent); 288 } 289 early_param("movablecore", cmdline_parse_movablecore); 290 291 /* 292 * early_calculate_totalpages() 293 * Sum pages in active regions for movable zone. 294 * Populate N_MEMORY for calculating usable_nodes. 295 */ 296 static unsigned long __init early_calculate_totalpages(void) 297 { 298 unsigned long totalpages = 0; 299 unsigned long start_pfn, end_pfn; 300 int i, nid; 301 302 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 303 unsigned long pages = end_pfn - start_pfn; 304 305 totalpages += pages; 306 if (pages) 307 node_set_state(nid, N_MEMORY); 308 } 309 return totalpages; 310 } 311 312 /* 313 * This finds a zone that can be used for ZONE_MOVABLE pages. The 314 * assumption is made that zones within a node are ordered in monotonic 315 * increasing memory addresses so that the "highest" populated zone is used 316 */ 317 static void __init find_usable_zone_for_movable(void) 318 { 319 int zone_index; 320 for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) { 321 if (zone_index == ZONE_MOVABLE) 322 continue; 323 324 if (arch_zone_highest_possible_pfn[zone_index] > 325 arch_zone_lowest_possible_pfn[zone_index]) 326 break; 327 } 328 329 VM_BUG_ON(zone_index == -1); 330 movable_zone = zone_index; 331 } 332 333 /* 334 * Find the PFN the Movable zone begins in each node. Kernel memory 335 * is spread evenly between nodes as long as the nodes have enough 336 * memory. When they don't, some nodes will have more kernelcore than 337 * others 338 */ 339 static void __init find_zone_movable_pfns_for_nodes(void) 340 { 341 int i, nid; 342 unsigned long usable_startpfn; 343 unsigned long kernelcore_node, kernelcore_remaining; 344 /* save the state before borrow the nodemask */ 345 nodemask_t saved_node_state = node_states[N_MEMORY]; 346 unsigned long totalpages = early_calculate_totalpages(); 347 int usable_nodes = nodes_weight(node_states[N_MEMORY]); 348 struct memblock_region *r; 349 350 /* Need to find movable_zone earlier when movable_node is specified. */ 351 find_usable_zone_for_movable(); 352 353 /* 354 * If movable_node is specified, ignore kernelcore and movablecore 355 * options. 356 */ 357 if (movable_node_is_enabled()) { 358 for_each_mem_region(r) { 359 if (!memblock_is_hotpluggable(r)) 360 continue; 361 362 nid = memblock_get_region_node(r); 363 364 usable_startpfn = PFN_DOWN(r->base); 365 zone_movable_pfn[nid] = zone_movable_pfn[nid] ? 366 min(usable_startpfn, zone_movable_pfn[nid]) : 367 usable_startpfn; 368 } 369 370 goto out2; 371 } 372 373 /* 374 * If kernelcore=mirror is specified, ignore movablecore option 375 */ 376 if (mirrored_kernelcore) { 377 bool mem_below_4gb_not_mirrored = false; 378 379 if (!memblock_has_mirror()) { 380 pr_warn("The system has no mirror memory, ignore kernelcore=mirror.\n"); 381 goto out; 382 } 383 384 for_each_mem_region(r) { 385 if (memblock_is_mirror(r)) 386 continue; 387 388 nid = memblock_get_region_node(r); 389 390 usable_startpfn = memblock_region_memory_base_pfn(r); 391 392 if (usable_startpfn < PHYS_PFN(SZ_4G)) { 393 mem_below_4gb_not_mirrored = true; 394 continue; 395 } 396 397 zone_movable_pfn[nid] = zone_movable_pfn[nid] ? 398 min(usable_startpfn, zone_movable_pfn[nid]) : 399 usable_startpfn; 400 } 401 402 if (mem_below_4gb_not_mirrored) 403 pr_warn("This configuration results in unmirrored kernel memory.\n"); 404 405 goto out2; 406 } 407 408 /* 409 * If kernelcore=nn% or movablecore=nn% was specified, calculate the 410 * amount of necessary memory. 411 */ 412 if (required_kernelcore_percent) 413 required_kernelcore = (totalpages * 100 * required_kernelcore_percent) / 414 10000UL; 415 if (required_movablecore_percent) 416 required_movablecore = (totalpages * 100 * required_movablecore_percent) / 417 10000UL; 418 419 /* 420 * If movablecore= was specified, calculate what size of 421 * kernelcore that corresponds so that memory usable for 422 * any allocation type is evenly spread. If both kernelcore 423 * and movablecore are specified, then the value of kernelcore 424 * will be used for required_kernelcore if it's greater than 425 * what movablecore would have allowed. 426 */ 427 if (required_movablecore) { 428 unsigned long corepages; 429 430 /* 431 * Round-up so that ZONE_MOVABLE is at least as large as what 432 * was requested by the user 433 */ 434 required_movablecore = 435 roundup(required_movablecore, MAX_ORDER_NR_PAGES); 436 required_movablecore = min(totalpages, required_movablecore); 437 corepages = totalpages - required_movablecore; 438 439 required_kernelcore = max(required_kernelcore, corepages); 440 } 441 442 /* 443 * If kernelcore was not specified or kernelcore size is larger 444 * than totalpages, there is no ZONE_MOVABLE. 445 */ 446 if (!required_kernelcore || required_kernelcore >= totalpages) 447 goto out; 448 449 /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */ 450 usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone]; 451 452 restart: 453 /* Spread kernelcore memory as evenly as possible throughout nodes */ 454 kernelcore_node = required_kernelcore / usable_nodes; 455 for_each_node_state(nid, N_MEMORY) { 456 unsigned long start_pfn, end_pfn; 457 458 /* 459 * Recalculate kernelcore_node if the division per node 460 * now exceeds what is necessary to satisfy the requested 461 * amount of memory for the kernel 462 */ 463 if (required_kernelcore < kernelcore_node) 464 kernelcore_node = required_kernelcore / usable_nodes; 465 466 /* 467 * As the map is walked, we track how much memory is usable 468 * by the kernel using kernelcore_remaining. When it is 469 * 0, the rest of the node is usable by ZONE_MOVABLE 470 */ 471 kernelcore_remaining = kernelcore_node; 472 473 /* Go through each range of PFNs within this node */ 474 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { 475 unsigned long size_pages; 476 477 start_pfn = max(start_pfn, zone_movable_pfn[nid]); 478 if (start_pfn >= end_pfn) 479 continue; 480 481 /* Account for what is only usable for kernelcore */ 482 if (start_pfn < usable_startpfn) { 483 unsigned long kernel_pages; 484 kernel_pages = min(end_pfn, usable_startpfn) 485 - start_pfn; 486 487 kernelcore_remaining -= min(kernel_pages, 488 kernelcore_remaining); 489 required_kernelcore -= min(kernel_pages, 490 required_kernelcore); 491 492 /* Continue if range is now fully accounted */ 493 if (end_pfn <= usable_startpfn) { 494 495 /* 496 * Push zone_movable_pfn to the end so 497 * that if we have to rebalance 498 * kernelcore across nodes, we will 499 * not double account here 500 */ 501 zone_movable_pfn[nid] = end_pfn; 502 continue; 503 } 504 start_pfn = usable_startpfn; 505 } 506 507 /* 508 * The usable PFN range for ZONE_MOVABLE is from 509 * start_pfn->end_pfn. Calculate size_pages as the 510 * number of pages used as kernelcore 511 */ 512 size_pages = end_pfn - start_pfn; 513 if (size_pages > kernelcore_remaining) 514 size_pages = kernelcore_remaining; 515 zone_movable_pfn[nid] = start_pfn + size_pages; 516 517 /* 518 * Some kernelcore has been met, update counts and 519 * break if the kernelcore for this node has been 520 * satisfied 521 */ 522 required_kernelcore -= min(required_kernelcore, 523 size_pages); 524 kernelcore_remaining -= size_pages; 525 if (!kernelcore_remaining) 526 break; 527 } 528 } 529 530 /* 531 * If there is still required_kernelcore, we do another pass with one 532 * less node in the count. This will push zone_movable_pfn[nid] further 533 * along on the nodes that still have memory until kernelcore is 534 * satisfied 535 */ 536 usable_nodes--; 537 if (usable_nodes && required_kernelcore > usable_nodes) 538 goto restart; 539 540 out2: 541 /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */ 542 for (nid = 0; nid < MAX_NUMNODES; nid++) { 543 unsigned long start_pfn, end_pfn; 544 545 zone_movable_pfn[nid] = 546 roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES); 547 548 get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); 549 if (zone_movable_pfn[nid] >= end_pfn) 550 zone_movable_pfn[nid] = 0; 551 } 552 553 out: 554 /* restore the node_state */ 555 node_states[N_MEMORY] = saved_node_state; 556 } 557 558 void __meminit __init_single_page(struct page *page, unsigned long pfn, 559 unsigned long zone, int nid) 560 { 561 mm_zero_struct_page(page); 562 set_page_links(page, zone, nid, pfn); 563 init_page_count(page); 564 page_mapcount_reset(page); 565 page_cpupid_reset_last(page); 566 page_kasan_tag_reset(page); 567 568 INIT_LIST_HEAD(&page->lru); 569 #ifdef WANT_PAGE_VIRTUAL 570 /* The shift won't overflow because ZONE_NORMAL is below 4G. */ 571 if (!is_highmem_idx(zone)) 572 set_page_address(page, __va(pfn << PAGE_SHIFT)); 573 #endif 574 } 575 576 #ifdef CONFIG_NUMA 577 /* 578 * During memory init memblocks map pfns to nids. The search is expensive and 579 * this caches recent lookups. The implementation of __early_pfn_to_nid 580 * treats start/end as pfns. 581 */ 582 struct mminit_pfnnid_cache { 583 unsigned long last_start; 584 unsigned long last_end; 585 int last_nid; 586 }; 587 588 static struct mminit_pfnnid_cache early_pfnnid_cache __meminitdata; 589 590 /* 591 * Required by SPARSEMEM. Given a PFN, return what node the PFN is on. 592 */ 593 static int __meminit __early_pfn_to_nid(unsigned long pfn, 594 struct mminit_pfnnid_cache *state) 595 { 596 unsigned long start_pfn, end_pfn; 597 int nid; 598 599 if (state->last_start <= pfn && pfn < state->last_end) 600 return state->last_nid; 601 602 nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn); 603 if (nid != NUMA_NO_NODE) { 604 state->last_start = start_pfn; 605 state->last_end = end_pfn; 606 state->last_nid = nid; 607 } 608 609 return nid; 610 } 611 612 int __meminit early_pfn_to_nid(unsigned long pfn) 613 { 614 static DEFINE_SPINLOCK(early_pfn_lock); 615 int nid; 616 617 spin_lock(&early_pfn_lock); 618 nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache); 619 if (nid < 0) 620 nid = first_online_node; 621 spin_unlock(&early_pfn_lock); 622 623 return nid; 624 } 625 626 int hashdist = HASHDIST_DEFAULT; 627 628 static int __init set_hashdist(char *str) 629 { 630 if (!str) 631 return 0; 632 hashdist = simple_strtoul(str, &str, 0); 633 return 1; 634 } 635 __setup("hashdist=", set_hashdist); 636 637 static inline void fixup_hashdist(void) 638 { 639 if (num_node_state(N_MEMORY) == 1) 640 hashdist = 0; 641 } 642 #else 643 static inline void fixup_hashdist(void) {} 644 #endif /* CONFIG_NUMA */ 645 646 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 647 static inline void pgdat_set_deferred_range(pg_data_t *pgdat) 648 { 649 pgdat->first_deferred_pfn = ULONG_MAX; 650 } 651 652 /* Returns true if the struct page for the pfn is initialised */ 653 static inline bool __meminit early_page_initialised(unsigned long pfn, int nid) 654 { 655 if (node_online(nid) && pfn >= NODE_DATA(nid)->first_deferred_pfn) 656 return false; 657 658 return true; 659 } 660 661 /* 662 * Returns true when the remaining initialisation should be deferred until 663 * later in the boot cycle when it can be parallelised. 664 */ 665 static bool __meminit 666 defer_init(int nid, unsigned long pfn, unsigned long end_pfn) 667 { 668 static unsigned long prev_end_pfn, nr_initialised; 669 670 if (early_page_ext_enabled()) 671 return false; 672 /* 673 * prev_end_pfn static that contains the end of previous zone 674 * No need to protect because called very early in boot before smp_init. 675 */ 676 if (prev_end_pfn != end_pfn) { 677 prev_end_pfn = end_pfn; 678 nr_initialised = 0; 679 } 680 681 /* Always populate low zones for address-constrained allocations */ 682 if (end_pfn < pgdat_end_pfn(NODE_DATA(nid))) 683 return false; 684 685 if (NODE_DATA(nid)->first_deferred_pfn != ULONG_MAX) 686 return true; 687 /* 688 * We start only with one section of pages, more pages are added as 689 * needed until the rest of deferred pages are initialized. 690 */ 691 nr_initialised++; 692 if ((nr_initialised > PAGES_PER_SECTION) && 693 (pfn & (PAGES_PER_SECTION - 1)) == 0) { 694 NODE_DATA(nid)->first_deferred_pfn = pfn; 695 return true; 696 } 697 return false; 698 } 699 700 static void __meminit init_reserved_page(unsigned long pfn, int nid) 701 { 702 pg_data_t *pgdat; 703 int zid; 704 705 if (early_page_initialised(pfn, nid)) 706 return; 707 708 pgdat = NODE_DATA(nid); 709 710 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 711 struct zone *zone = &pgdat->node_zones[zid]; 712 713 if (zone_spans_pfn(zone, pfn)) 714 break; 715 } 716 __init_single_page(pfn_to_page(pfn), pfn, zid, nid); 717 } 718 #else 719 static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {} 720 721 static inline bool early_page_initialised(unsigned long pfn, int nid) 722 { 723 return true; 724 } 725 726 static inline bool defer_init(int nid, unsigned long pfn, unsigned long end_pfn) 727 { 728 return false; 729 } 730 731 static inline void init_reserved_page(unsigned long pfn, int nid) 732 { 733 } 734 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ 735 736 /* 737 * Initialised pages do not have PageReserved set. This function is 738 * called for each range allocated by the bootmem allocator and 739 * marks the pages PageReserved. The remaining valid pages are later 740 * sent to the buddy page allocator. 741 */ 742 void __meminit reserve_bootmem_region(phys_addr_t start, 743 phys_addr_t end, int nid) 744 { 745 unsigned long start_pfn = PFN_DOWN(start); 746 unsigned long end_pfn = PFN_UP(end); 747 748 for (; start_pfn < end_pfn; start_pfn++) { 749 if (pfn_valid(start_pfn)) { 750 struct page *page = pfn_to_page(start_pfn); 751 752 init_reserved_page(start_pfn, nid); 753 754 /* Avoid false-positive PageTail() */ 755 INIT_LIST_HEAD(&page->lru); 756 757 /* 758 * no need for atomic set_bit because the struct 759 * page is not visible yet so nobody should 760 * access it yet. 761 */ 762 __SetPageReserved(page); 763 } 764 } 765 } 766 767 /* If zone is ZONE_MOVABLE but memory is mirrored, it is an overlapped init */ 768 static bool __meminit 769 overlap_memmap_init(unsigned long zone, unsigned long *pfn) 770 { 771 static struct memblock_region *r; 772 773 if (mirrored_kernelcore && zone == ZONE_MOVABLE) { 774 if (!r || *pfn >= memblock_region_memory_end_pfn(r)) { 775 for_each_mem_region(r) { 776 if (*pfn < memblock_region_memory_end_pfn(r)) 777 break; 778 } 779 } 780 if (*pfn >= memblock_region_memory_base_pfn(r) && 781 memblock_is_mirror(r)) { 782 *pfn = memblock_region_memory_end_pfn(r); 783 return true; 784 } 785 } 786 return false; 787 } 788 789 /* 790 * Only struct pages that correspond to ranges defined by memblock.memory 791 * are zeroed and initialized by going through __init_single_page() during 792 * memmap_init_zone_range(). 793 * 794 * But, there could be struct pages that correspond to holes in 795 * memblock.memory. This can happen because of the following reasons: 796 * - physical memory bank size is not necessarily the exact multiple of the 797 * arbitrary section size 798 * - early reserved memory may not be listed in memblock.memory 799 * - non-memory regions covered by the contigious flatmem mapping 800 * - memory layouts defined with memmap= kernel parameter may not align 801 * nicely with memmap sections 802 * 803 * Explicitly initialize those struct pages so that: 804 * - PG_Reserved is set 805 * - zone and node links point to zone and node that span the page if the 806 * hole is in the middle of a zone 807 * - zone and node links point to adjacent zone/node if the hole falls on 808 * the zone boundary; the pages in such holes will be prepended to the 809 * zone/node above the hole except for the trailing pages in the last 810 * section that will be appended to the zone/node below. 811 */ 812 static void __init init_unavailable_range(unsigned long spfn, 813 unsigned long epfn, 814 int zone, int node) 815 { 816 unsigned long pfn; 817 u64 pgcnt = 0; 818 819 for (pfn = spfn; pfn < epfn; pfn++) { 820 if (!pfn_valid(pageblock_start_pfn(pfn))) { 821 pfn = pageblock_end_pfn(pfn) - 1; 822 continue; 823 } 824 __init_single_page(pfn_to_page(pfn), pfn, zone, node); 825 __SetPageReserved(pfn_to_page(pfn)); 826 pgcnt++; 827 } 828 829 if (pgcnt) 830 pr_info("On node %d, zone %s: %lld pages in unavailable ranges\n", 831 node, zone_names[zone], pgcnt); 832 } 833 834 /* 835 * Initially all pages are reserved - free ones are freed 836 * up by memblock_free_all() once the early boot process is 837 * done. Non-atomic initialization, single-pass. 838 * 839 * All aligned pageblocks are initialized to the specified migratetype 840 * (usually MIGRATE_MOVABLE). Besides setting the migratetype, no related 841 * zone stats (e.g., nr_isolate_pageblock) are touched. 842 */ 843 void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone, 844 unsigned long start_pfn, unsigned long zone_end_pfn, 845 enum meminit_context context, 846 struct vmem_altmap *altmap, int migratetype) 847 { 848 unsigned long pfn, end_pfn = start_pfn + size; 849 struct page *page; 850 851 if (highest_memmap_pfn < end_pfn - 1) 852 highest_memmap_pfn = end_pfn - 1; 853 854 #ifdef CONFIG_ZONE_DEVICE 855 /* 856 * Honor reservation requested by the driver for this ZONE_DEVICE 857 * memory. We limit the total number of pages to initialize to just 858 * those that might contain the memory mapping. We will defer the 859 * ZONE_DEVICE page initialization until after we have released 860 * the hotplug lock. 861 */ 862 if (zone == ZONE_DEVICE) { 863 if (!altmap) 864 return; 865 866 if (start_pfn == altmap->base_pfn) 867 start_pfn += altmap->reserve; 868 end_pfn = altmap->base_pfn + vmem_altmap_offset(altmap); 869 } 870 #endif 871 872 for (pfn = start_pfn; pfn < end_pfn; ) { 873 /* 874 * There can be holes in boot-time mem_map[]s handed to this 875 * function. They do not exist on hotplugged memory. 876 */ 877 if (context == MEMINIT_EARLY) { 878 if (overlap_memmap_init(zone, &pfn)) 879 continue; 880 if (defer_init(nid, pfn, zone_end_pfn)) { 881 deferred_struct_pages = true; 882 break; 883 } 884 } 885 886 page = pfn_to_page(pfn); 887 __init_single_page(page, pfn, zone, nid); 888 if (context == MEMINIT_HOTPLUG) 889 __SetPageReserved(page); 890 891 /* 892 * Usually, we want to mark the pageblock MIGRATE_MOVABLE, 893 * such that unmovable allocations won't be scattered all 894 * over the place during system boot. 895 */ 896 if (pageblock_aligned(pfn)) { 897 set_pageblock_migratetype(page, migratetype); 898 cond_resched(); 899 } 900 pfn++; 901 } 902 } 903 904 static void __init memmap_init_zone_range(struct zone *zone, 905 unsigned long start_pfn, 906 unsigned long end_pfn, 907 unsigned long *hole_pfn) 908 { 909 unsigned long zone_start_pfn = zone->zone_start_pfn; 910 unsigned long zone_end_pfn = zone_start_pfn + zone->spanned_pages; 911 int nid = zone_to_nid(zone), zone_id = zone_idx(zone); 912 913 start_pfn = clamp(start_pfn, zone_start_pfn, zone_end_pfn); 914 end_pfn = clamp(end_pfn, zone_start_pfn, zone_end_pfn); 915 916 if (start_pfn >= end_pfn) 917 return; 918 919 memmap_init_range(end_pfn - start_pfn, nid, zone_id, start_pfn, 920 zone_end_pfn, MEMINIT_EARLY, NULL, MIGRATE_MOVABLE); 921 922 if (*hole_pfn < start_pfn) 923 init_unavailable_range(*hole_pfn, start_pfn, zone_id, nid); 924 925 *hole_pfn = end_pfn; 926 } 927 928 static void __init memmap_init(void) 929 { 930 unsigned long start_pfn, end_pfn; 931 unsigned long hole_pfn = 0; 932 int i, j, zone_id = 0, nid; 933 934 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 935 struct pglist_data *node = NODE_DATA(nid); 936 937 for (j = 0; j < MAX_NR_ZONES; j++) { 938 struct zone *zone = node->node_zones + j; 939 940 if (!populated_zone(zone)) 941 continue; 942 943 memmap_init_zone_range(zone, start_pfn, end_pfn, 944 &hole_pfn); 945 zone_id = j; 946 } 947 } 948 949 #ifdef CONFIG_SPARSEMEM 950 /* 951 * Initialize the memory map for hole in the range [memory_end, 952 * section_end]. 953 * Append the pages in this hole to the highest zone in the last 954 * node. 955 * The call to init_unavailable_range() is outside the ifdef to 956 * silence the compiler warining about zone_id set but not used; 957 * for FLATMEM it is a nop anyway 958 */ 959 end_pfn = round_up(end_pfn, PAGES_PER_SECTION); 960 if (hole_pfn < end_pfn) 961 #endif 962 init_unavailable_range(hole_pfn, end_pfn, zone_id, nid); 963 } 964 965 #ifdef CONFIG_ZONE_DEVICE 966 static void __ref __init_zone_device_page(struct page *page, unsigned long pfn, 967 unsigned long zone_idx, int nid, 968 struct dev_pagemap *pgmap) 969 { 970 971 __init_single_page(page, pfn, zone_idx, nid); 972 973 /* 974 * Mark page reserved as it will need to wait for onlining 975 * phase for it to be fully associated with a zone. 976 * 977 * We can use the non-atomic __set_bit operation for setting 978 * the flag as we are still initializing the pages. 979 */ 980 __SetPageReserved(page); 981 982 /* 983 * ZONE_DEVICE pages union ->lru with a ->pgmap back pointer 984 * and zone_device_data. It is a bug if a ZONE_DEVICE page is 985 * ever freed or placed on a driver-private list. 986 */ 987 page->pgmap = pgmap; 988 page->zone_device_data = NULL; 989 990 /* 991 * Mark the block movable so that blocks are reserved for 992 * movable at startup. This will force kernel allocations 993 * to reserve their blocks rather than leaking throughout 994 * the address space during boot when many long-lived 995 * kernel allocations are made. 996 * 997 * Please note that MEMINIT_HOTPLUG path doesn't clear memmap 998 * because this is done early in section_activate() 999 */ 1000 if (pageblock_aligned(pfn)) { 1001 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 1002 cond_resched(); 1003 } 1004 1005 /* 1006 * ZONE_DEVICE pages are released directly to the driver page allocator 1007 * which will set the page count to 1 when allocating the page. 1008 */ 1009 if (pgmap->type == MEMORY_DEVICE_PRIVATE || 1010 pgmap->type == MEMORY_DEVICE_COHERENT) 1011 set_page_count(page, 0); 1012 } 1013 1014 /* 1015 * With compound page geometry and when struct pages are stored in ram most 1016 * tail pages are reused. Consequently, the amount of unique struct pages to 1017 * initialize is a lot smaller that the total amount of struct pages being 1018 * mapped. This is a paired / mild layering violation with explicit knowledge 1019 * of how the sparse_vmemmap internals handle compound pages in the lack 1020 * of an altmap. See vmemmap_populate_compound_pages(). 1021 */ 1022 static inline unsigned long compound_nr_pages(struct vmem_altmap *altmap, 1023 struct dev_pagemap *pgmap) 1024 { 1025 if (!vmemmap_can_optimize(altmap, pgmap)) 1026 return pgmap_vmemmap_nr(pgmap); 1027 1028 return VMEMMAP_RESERVE_NR * (PAGE_SIZE / sizeof(struct page)); 1029 } 1030 1031 static void __ref memmap_init_compound(struct page *head, 1032 unsigned long head_pfn, 1033 unsigned long zone_idx, int nid, 1034 struct dev_pagemap *pgmap, 1035 unsigned long nr_pages) 1036 { 1037 unsigned long pfn, end_pfn = head_pfn + nr_pages; 1038 unsigned int order = pgmap->vmemmap_shift; 1039 1040 __SetPageHead(head); 1041 for (pfn = head_pfn + 1; pfn < end_pfn; pfn++) { 1042 struct page *page = pfn_to_page(pfn); 1043 1044 __init_zone_device_page(page, pfn, zone_idx, nid, pgmap); 1045 prep_compound_tail(head, pfn - head_pfn); 1046 set_page_count(page, 0); 1047 1048 /* 1049 * The first tail page stores important compound page info. 1050 * Call prep_compound_head() after the first tail page has 1051 * been initialized, to not have the data overwritten. 1052 */ 1053 if (pfn == head_pfn + 1) 1054 prep_compound_head(head, order); 1055 } 1056 } 1057 1058 void __ref memmap_init_zone_device(struct zone *zone, 1059 unsigned long start_pfn, 1060 unsigned long nr_pages, 1061 struct dev_pagemap *pgmap) 1062 { 1063 unsigned long pfn, end_pfn = start_pfn + nr_pages; 1064 struct pglist_data *pgdat = zone->zone_pgdat; 1065 struct vmem_altmap *altmap = pgmap_altmap(pgmap); 1066 unsigned int pfns_per_compound = pgmap_vmemmap_nr(pgmap); 1067 unsigned long zone_idx = zone_idx(zone); 1068 unsigned long start = jiffies; 1069 int nid = pgdat->node_id; 1070 1071 if (WARN_ON_ONCE(!pgmap || zone_idx != ZONE_DEVICE)) 1072 return; 1073 1074 /* 1075 * The call to memmap_init should have already taken care 1076 * of the pages reserved for the memmap, so we can just jump to 1077 * the end of that region and start processing the device pages. 1078 */ 1079 if (altmap) { 1080 start_pfn = altmap->base_pfn + vmem_altmap_offset(altmap); 1081 nr_pages = end_pfn - start_pfn; 1082 } 1083 1084 for (pfn = start_pfn; pfn < end_pfn; pfn += pfns_per_compound) { 1085 struct page *page = pfn_to_page(pfn); 1086 1087 __init_zone_device_page(page, pfn, zone_idx, nid, pgmap); 1088 1089 if (pfns_per_compound == 1) 1090 continue; 1091 1092 memmap_init_compound(page, pfn, zone_idx, nid, pgmap, 1093 compound_nr_pages(altmap, pgmap)); 1094 } 1095 1096 pr_debug("%s initialised %lu pages in %ums\n", __func__, 1097 nr_pages, jiffies_to_msecs(jiffies - start)); 1098 } 1099 #endif 1100 1101 /* 1102 * The zone ranges provided by the architecture do not include ZONE_MOVABLE 1103 * because it is sized independent of architecture. Unlike the other zones, 1104 * the starting point for ZONE_MOVABLE is not fixed. It may be different 1105 * in each node depending on the size of each node and how evenly kernelcore 1106 * is distributed. This helper function adjusts the zone ranges 1107 * provided by the architecture for a given node by using the end of the 1108 * highest usable zone for ZONE_MOVABLE. This preserves the assumption that 1109 * zones within a node are in order of monotonic increases memory addresses 1110 */ 1111 static void __init adjust_zone_range_for_zone_movable(int nid, 1112 unsigned long zone_type, 1113 unsigned long node_end_pfn, 1114 unsigned long *zone_start_pfn, 1115 unsigned long *zone_end_pfn) 1116 { 1117 /* Only adjust if ZONE_MOVABLE is on this node */ 1118 if (zone_movable_pfn[nid]) { 1119 /* Size ZONE_MOVABLE */ 1120 if (zone_type == ZONE_MOVABLE) { 1121 *zone_start_pfn = zone_movable_pfn[nid]; 1122 *zone_end_pfn = min(node_end_pfn, 1123 arch_zone_highest_possible_pfn[movable_zone]); 1124 1125 /* Adjust for ZONE_MOVABLE starting within this range */ 1126 } else if (!mirrored_kernelcore && 1127 *zone_start_pfn < zone_movable_pfn[nid] && 1128 *zone_end_pfn > zone_movable_pfn[nid]) { 1129 *zone_end_pfn = zone_movable_pfn[nid]; 1130 1131 /* Check if this whole range is within ZONE_MOVABLE */ 1132 } else if (*zone_start_pfn >= zone_movable_pfn[nid]) 1133 *zone_start_pfn = *zone_end_pfn; 1134 } 1135 } 1136 1137 /* 1138 * Return the number of holes in a range on a node. If nid is MAX_NUMNODES, 1139 * then all holes in the requested range will be accounted for. 1140 */ 1141 unsigned long __init __absent_pages_in_range(int nid, 1142 unsigned long range_start_pfn, 1143 unsigned long range_end_pfn) 1144 { 1145 unsigned long nr_absent = range_end_pfn - range_start_pfn; 1146 unsigned long start_pfn, end_pfn; 1147 int i; 1148 1149 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { 1150 start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn); 1151 end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn); 1152 nr_absent -= end_pfn - start_pfn; 1153 } 1154 return nr_absent; 1155 } 1156 1157 /** 1158 * absent_pages_in_range - Return number of page frames in holes within a range 1159 * @start_pfn: The start PFN to start searching for holes 1160 * @end_pfn: The end PFN to stop searching for holes 1161 * 1162 * Return: the number of pages frames in memory holes within a range. 1163 */ 1164 unsigned long __init absent_pages_in_range(unsigned long start_pfn, 1165 unsigned long end_pfn) 1166 { 1167 return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn); 1168 } 1169 1170 /* Return the number of page frames in holes in a zone on a node */ 1171 static unsigned long __init zone_absent_pages_in_node(int nid, 1172 unsigned long zone_type, 1173 unsigned long zone_start_pfn, 1174 unsigned long zone_end_pfn) 1175 { 1176 unsigned long nr_absent; 1177 1178 /* zone is empty, we don't have any absent pages */ 1179 if (zone_start_pfn == zone_end_pfn) 1180 return 0; 1181 1182 nr_absent = __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn); 1183 1184 /* 1185 * ZONE_MOVABLE handling. 1186 * Treat pages to be ZONE_MOVABLE in ZONE_NORMAL as absent pages 1187 * and vice versa. 1188 */ 1189 if (mirrored_kernelcore && zone_movable_pfn[nid]) { 1190 unsigned long start_pfn, end_pfn; 1191 struct memblock_region *r; 1192 1193 for_each_mem_region(r) { 1194 start_pfn = clamp(memblock_region_memory_base_pfn(r), 1195 zone_start_pfn, zone_end_pfn); 1196 end_pfn = clamp(memblock_region_memory_end_pfn(r), 1197 zone_start_pfn, zone_end_pfn); 1198 1199 if (zone_type == ZONE_MOVABLE && 1200 memblock_is_mirror(r)) 1201 nr_absent += end_pfn - start_pfn; 1202 1203 if (zone_type == ZONE_NORMAL && 1204 !memblock_is_mirror(r)) 1205 nr_absent += end_pfn - start_pfn; 1206 } 1207 } 1208 1209 return nr_absent; 1210 } 1211 1212 /* 1213 * Return the number of pages a zone spans in a node, including holes 1214 * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node() 1215 */ 1216 static unsigned long __init zone_spanned_pages_in_node(int nid, 1217 unsigned long zone_type, 1218 unsigned long node_start_pfn, 1219 unsigned long node_end_pfn, 1220 unsigned long *zone_start_pfn, 1221 unsigned long *zone_end_pfn) 1222 { 1223 unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type]; 1224 unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type]; 1225 1226 /* Get the start and end of the zone */ 1227 *zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high); 1228 *zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high); 1229 adjust_zone_range_for_zone_movable(nid, zone_type, node_end_pfn, 1230 zone_start_pfn, zone_end_pfn); 1231 1232 /* Check that this node has pages within the zone's required range */ 1233 if (*zone_end_pfn < node_start_pfn || *zone_start_pfn > node_end_pfn) 1234 return 0; 1235 1236 /* Move the zone boundaries inside the node if necessary */ 1237 *zone_end_pfn = min(*zone_end_pfn, node_end_pfn); 1238 *zone_start_pfn = max(*zone_start_pfn, node_start_pfn); 1239 1240 /* Return the spanned pages */ 1241 return *zone_end_pfn - *zone_start_pfn; 1242 } 1243 1244 static void __init reset_memoryless_node_totalpages(struct pglist_data *pgdat) 1245 { 1246 struct zone *z; 1247 1248 for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++) { 1249 z->zone_start_pfn = 0; 1250 z->spanned_pages = 0; 1251 z->present_pages = 0; 1252 #if defined(CONFIG_MEMORY_HOTPLUG) 1253 z->present_early_pages = 0; 1254 #endif 1255 } 1256 1257 pgdat->node_spanned_pages = 0; 1258 pgdat->node_present_pages = 0; 1259 pr_debug("On node %d totalpages: 0\n", pgdat->node_id); 1260 } 1261 1262 static void __init calculate_node_totalpages(struct pglist_data *pgdat, 1263 unsigned long node_start_pfn, 1264 unsigned long node_end_pfn) 1265 { 1266 unsigned long realtotalpages = 0, totalpages = 0; 1267 enum zone_type i; 1268 1269 for (i = 0; i < MAX_NR_ZONES; i++) { 1270 struct zone *zone = pgdat->node_zones + i; 1271 unsigned long zone_start_pfn, zone_end_pfn; 1272 unsigned long spanned, absent; 1273 unsigned long real_size; 1274 1275 spanned = zone_spanned_pages_in_node(pgdat->node_id, i, 1276 node_start_pfn, 1277 node_end_pfn, 1278 &zone_start_pfn, 1279 &zone_end_pfn); 1280 absent = zone_absent_pages_in_node(pgdat->node_id, i, 1281 zone_start_pfn, 1282 zone_end_pfn); 1283 1284 real_size = spanned - absent; 1285 1286 if (spanned) 1287 zone->zone_start_pfn = zone_start_pfn; 1288 else 1289 zone->zone_start_pfn = 0; 1290 zone->spanned_pages = spanned; 1291 zone->present_pages = real_size; 1292 #if defined(CONFIG_MEMORY_HOTPLUG) 1293 zone->present_early_pages = real_size; 1294 #endif 1295 1296 totalpages += spanned; 1297 realtotalpages += real_size; 1298 } 1299 1300 pgdat->node_spanned_pages = totalpages; 1301 pgdat->node_present_pages = realtotalpages; 1302 pr_debug("On node %d totalpages: %lu\n", pgdat->node_id, realtotalpages); 1303 } 1304 1305 static unsigned long __init calc_memmap_size(unsigned long spanned_pages, 1306 unsigned long present_pages) 1307 { 1308 unsigned long pages = spanned_pages; 1309 1310 /* 1311 * Provide a more accurate estimation if there are holes within 1312 * the zone and SPARSEMEM is in use. If there are holes within the 1313 * zone, each populated memory region may cost us one or two extra 1314 * memmap pages due to alignment because memmap pages for each 1315 * populated regions may not be naturally aligned on page boundary. 1316 * So the (present_pages >> 4) heuristic is a tradeoff for that. 1317 */ 1318 if (spanned_pages > present_pages + (present_pages >> 4) && 1319 IS_ENABLED(CONFIG_SPARSEMEM)) 1320 pages = present_pages; 1321 1322 return PAGE_ALIGN(pages * sizeof(struct page)) >> PAGE_SHIFT; 1323 } 1324 1325 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1326 static void pgdat_init_split_queue(struct pglist_data *pgdat) 1327 { 1328 struct deferred_split *ds_queue = &pgdat->deferred_split_queue; 1329 1330 spin_lock_init(&ds_queue->split_queue_lock); 1331 INIT_LIST_HEAD(&ds_queue->split_queue); 1332 ds_queue->split_queue_len = 0; 1333 } 1334 #else 1335 static void pgdat_init_split_queue(struct pglist_data *pgdat) {} 1336 #endif 1337 1338 #ifdef CONFIG_COMPACTION 1339 static void pgdat_init_kcompactd(struct pglist_data *pgdat) 1340 { 1341 init_waitqueue_head(&pgdat->kcompactd_wait); 1342 } 1343 #else 1344 static void pgdat_init_kcompactd(struct pglist_data *pgdat) {} 1345 #endif 1346 1347 static void __meminit pgdat_init_internals(struct pglist_data *pgdat) 1348 { 1349 int i; 1350 1351 pgdat_resize_init(pgdat); 1352 pgdat_kswapd_lock_init(pgdat); 1353 1354 pgdat_init_split_queue(pgdat); 1355 pgdat_init_kcompactd(pgdat); 1356 1357 init_waitqueue_head(&pgdat->kswapd_wait); 1358 init_waitqueue_head(&pgdat->pfmemalloc_wait); 1359 1360 for (i = 0; i < NR_VMSCAN_THROTTLE; i++) 1361 init_waitqueue_head(&pgdat->reclaim_wait[i]); 1362 1363 pgdat_page_ext_init(pgdat); 1364 lruvec_init(&pgdat->__lruvec); 1365 } 1366 1367 static void __meminit zone_init_internals(struct zone *zone, enum zone_type idx, int nid, 1368 unsigned long remaining_pages) 1369 { 1370 atomic_long_set(&zone->managed_pages, remaining_pages); 1371 zone_set_nid(zone, nid); 1372 zone->name = zone_names[idx]; 1373 zone->zone_pgdat = NODE_DATA(nid); 1374 spin_lock_init(&zone->lock); 1375 zone_seqlock_init(zone); 1376 zone_pcp_init(zone); 1377 } 1378 1379 static void __meminit zone_init_free_lists(struct zone *zone) 1380 { 1381 unsigned int order, t; 1382 for_each_migratetype_order(order, t) { 1383 INIT_LIST_HEAD(&zone->free_area[order].free_list[t]); 1384 zone->free_area[order].nr_free = 0; 1385 } 1386 1387 #ifdef CONFIG_UNACCEPTED_MEMORY 1388 INIT_LIST_HEAD(&zone->unaccepted_pages); 1389 #endif 1390 } 1391 1392 void __meminit init_currently_empty_zone(struct zone *zone, 1393 unsigned long zone_start_pfn, 1394 unsigned long size) 1395 { 1396 struct pglist_data *pgdat = zone->zone_pgdat; 1397 int zone_idx = zone_idx(zone) + 1; 1398 1399 if (zone_idx > pgdat->nr_zones) 1400 pgdat->nr_zones = zone_idx; 1401 1402 zone->zone_start_pfn = zone_start_pfn; 1403 1404 mminit_dprintk(MMINIT_TRACE, "memmap_init", 1405 "Initialising map node %d zone %lu pfns %lu -> %lu\n", 1406 pgdat->node_id, 1407 (unsigned long)zone_idx(zone), 1408 zone_start_pfn, (zone_start_pfn + size)); 1409 1410 zone_init_free_lists(zone); 1411 zone->initialized = 1; 1412 } 1413 1414 #ifndef CONFIG_SPARSEMEM 1415 /* 1416 * Calculate the size of the zone->blockflags rounded to an unsigned long 1417 * Start by making sure zonesize is a multiple of pageblock_order by rounding 1418 * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally 1419 * round what is now in bits to nearest long in bits, then return it in 1420 * bytes. 1421 */ 1422 static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize) 1423 { 1424 unsigned long usemapsize; 1425 1426 zonesize += zone_start_pfn & (pageblock_nr_pages-1); 1427 usemapsize = roundup(zonesize, pageblock_nr_pages); 1428 usemapsize = usemapsize >> pageblock_order; 1429 usemapsize *= NR_PAGEBLOCK_BITS; 1430 usemapsize = roundup(usemapsize, BITS_PER_LONG); 1431 1432 return usemapsize / BITS_PER_BYTE; 1433 } 1434 1435 static void __ref setup_usemap(struct zone *zone) 1436 { 1437 unsigned long usemapsize = usemap_size(zone->zone_start_pfn, 1438 zone->spanned_pages); 1439 zone->pageblock_flags = NULL; 1440 if (usemapsize) { 1441 zone->pageblock_flags = 1442 memblock_alloc_node(usemapsize, SMP_CACHE_BYTES, 1443 zone_to_nid(zone)); 1444 if (!zone->pageblock_flags) 1445 panic("Failed to allocate %ld bytes for zone %s pageblock flags on node %d\n", 1446 usemapsize, zone->name, zone_to_nid(zone)); 1447 } 1448 } 1449 #else 1450 static inline void setup_usemap(struct zone *zone) {} 1451 #endif /* CONFIG_SPARSEMEM */ 1452 1453 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE 1454 1455 /* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */ 1456 void __init set_pageblock_order(void) 1457 { 1458 unsigned int order = MAX_PAGE_ORDER; 1459 1460 /* Check that pageblock_nr_pages has not already been setup */ 1461 if (pageblock_order) 1462 return; 1463 1464 /* Don't let pageblocks exceed the maximum allocation granularity. */ 1465 if (HPAGE_SHIFT > PAGE_SHIFT && HUGETLB_PAGE_ORDER < order) 1466 order = HUGETLB_PAGE_ORDER; 1467 1468 /* 1469 * Assume the largest contiguous order of interest is a huge page. 1470 * This value may be variable depending on boot parameters on powerpc. 1471 */ 1472 pageblock_order = order; 1473 } 1474 #else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ 1475 1476 /* 1477 * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order() 1478 * is unused as pageblock_order is set at compile-time. See 1479 * include/linux/pageblock-flags.h for the values of pageblock_order based on 1480 * the kernel config 1481 */ 1482 void __init set_pageblock_order(void) 1483 { 1484 } 1485 1486 #endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ 1487 1488 /* 1489 * Set up the zone data structures 1490 * - init pgdat internals 1491 * - init all zones belonging to this node 1492 * 1493 * NOTE: this function is only called during memory hotplug 1494 */ 1495 #ifdef CONFIG_MEMORY_HOTPLUG 1496 void __ref free_area_init_core_hotplug(struct pglist_data *pgdat) 1497 { 1498 int nid = pgdat->node_id; 1499 enum zone_type z; 1500 int cpu; 1501 1502 pgdat_init_internals(pgdat); 1503 1504 if (pgdat->per_cpu_nodestats == &boot_nodestats) 1505 pgdat->per_cpu_nodestats = alloc_percpu(struct per_cpu_nodestat); 1506 1507 /* 1508 * Reset the nr_zones, order and highest_zoneidx before reuse. 1509 * Note that kswapd will init kswapd_highest_zoneidx properly 1510 * when it starts in the near future. 1511 */ 1512 pgdat->nr_zones = 0; 1513 pgdat->kswapd_order = 0; 1514 pgdat->kswapd_highest_zoneidx = 0; 1515 pgdat->node_start_pfn = 0; 1516 pgdat->node_present_pages = 0; 1517 1518 for_each_online_cpu(cpu) { 1519 struct per_cpu_nodestat *p; 1520 1521 p = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu); 1522 memset(p, 0, sizeof(*p)); 1523 } 1524 1525 /* 1526 * When memory is hot-added, all the memory is in offline state. So 1527 * clear all zones' present_pages and managed_pages because they will 1528 * be updated in online_pages() and offline_pages(). 1529 */ 1530 for (z = 0; z < MAX_NR_ZONES; z++) { 1531 struct zone *zone = pgdat->node_zones + z; 1532 1533 zone->present_pages = 0; 1534 zone_init_internals(zone, z, nid, 0); 1535 } 1536 } 1537 #endif 1538 1539 /* 1540 * Set up the zone data structures: 1541 * - mark all pages reserved 1542 * - mark all memory queues empty 1543 * - clear the memory bitmaps 1544 * 1545 * NOTE: pgdat should get zeroed by caller. 1546 * NOTE: this function is only called during early init. 1547 */ 1548 static void __init free_area_init_core(struct pglist_data *pgdat) 1549 { 1550 enum zone_type j; 1551 int nid = pgdat->node_id; 1552 1553 pgdat_init_internals(pgdat); 1554 pgdat->per_cpu_nodestats = &boot_nodestats; 1555 1556 for (j = 0; j < MAX_NR_ZONES; j++) { 1557 struct zone *zone = pgdat->node_zones + j; 1558 unsigned long size, freesize, memmap_pages; 1559 1560 size = zone->spanned_pages; 1561 freesize = zone->present_pages; 1562 1563 /* 1564 * Adjust freesize so that it accounts for how much memory 1565 * is used by this zone for memmap. This affects the watermark 1566 * and per-cpu initialisations 1567 */ 1568 memmap_pages = calc_memmap_size(size, freesize); 1569 if (!is_highmem_idx(j)) { 1570 if (freesize >= memmap_pages) { 1571 freesize -= memmap_pages; 1572 if (memmap_pages) 1573 pr_debug(" %s zone: %lu pages used for memmap\n", 1574 zone_names[j], memmap_pages); 1575 } else 1576 pr_warn(" %s zone: %lu memmap pages exceeds freesize %lu\n", 1577 zone_names[j], memmap_pages, freesize); 1578 } 1579 1580 /* Account for reserved pages */ 1581 if (j == 0 && freesize > dma_reserve) { 1582 freesize -= dma_reserve; 1583 pr_debug(" %s zone: %lu pages reserved\n", zone_names[0], dma_reserve); 1584 } 1585 1586 if (!is_highmem_idx(j)) 1587 nr_kernel_pages += freesize; 1588 /* Charge for highmem memmap if there are enough kernel pages */ 1589 else if (nr_kernel_pages > memmap_pages * 2) 1590 nr_kernel_pages -= memmap_pages; 1591 nr_all_pages += freesize; 1592 1593 /* 1594 * Set an approximate value for lowmem here, it will be adjusted 1595 * when the bootmem allocator frees pages into the buddy system. 1596 * And all highmem pages will be managed by the buddy system. 1597 */ 1598 zone_init_internals(zone, j, nid, freesize); 1599 1600 if (!size) 1601 continue; 1602 1603 setup_usemap(zone); 1604 init_currently_empty_zone(zone, zone->zone_start_pfn, size); 1605 } 1606 } 1607 1608 void __init *memmap_alloc(phys_addr_t size, phys_addr_t align, 1609 phys_addr_t min_addr, int nid, bool exact_nid) 1610 { 1611 void *ptr; 1612 1613 if (exact_nid) 1614 ptr = memblock_alloc_exact_nid_raw(size, align, min_addr, 1615 MEMBLOCK_ALLOC_ACCESSIBLE, 1616 nid); 1617 else 1618 ptr = memblock_alloc_try_nid_raw(size, align, min_addr, 1619 MEMBLOCK_ALLOC_ACCESSIBLE, 1620 nid); 1621 1622 if (ptr && size > 0) 1623 page_init_poison(ptr, size); 1624 1625 return ptr; 1626 } 1627 1628 #ifdef CONFIG_FLATMEM 1629 static void __init alloc_node_mem_map(struct pglist_data *pgdat) 1630 { 1631 unsigned long start, offset, size, end; 1632 struct page *map; 1633 1634 /* Skip empty nodes */ 1635 if (!pgdat->node_spanned_pages) 1636 return; 1637 1638 start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1); 1639 offset = pgdat->node_start_pfn - start; 1640 /* 1641 * The zone's endpoints aren't required to be MAX_PAGE_ORDER 1642 * aligned but the node_mem_map endpoints must be in order 1643 * for the buddy allocator to function correctly. 1644 */ 1645 end = ALIGN(pgdat_end_pfn(pgdat), MAX_ORDER_NR_PAGES); 1646 size = (end - start) * sizeof(struct page); 1647 map = memmap_alloc(size, SMP_CACHE_BYTES, MEMBLOCK_LOW_LIMIT, 1648 pgdat->node_id, false); 1649 if (!map) 1650 panic("Failed to allocate %ld bytes for node %d memory map\n", 1651 size, pgdat->node_id); 1652 pgdat->node_mem_map = map + offset; 1653 pr_debug("%s: node %d, pgdat %08lx, node_mem_map %08lx\n", 1654 __func__, pgdat->node_id, (unsigned long)pgdat, 1655 (unsigned long)pgdat->node_mem_map); 1656 #ifndef CONFIG_NUMA 1657 /* the global mem_map is just set as node 0's */ 1658 if (pgdat == NODE_DATA(0)) { 1659 mem_map = NODE_DATA(0)->node_mem_map; 1660 if (page_to_pfn(mem_map) != pgdat->node_start_pfn) 1661 mem_map -= offset; 1662 } 1663 #endif 1664 } 1665 #else 1666 static inline void alloc_node_mem_map(struct pglist_data *pgdat) { } 1667 #endif /* CONFIG_FLATMEM */ 1668 1669 /** 1670 * get_pfn_range_for_nid - Return the start and end page frames for a node 1671 * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned. 1672 * @start_pfn: Passed by reference. On return, it will have the node start_pfn. 1673 * @end_pfn: Passed by reference. On return, it will have the node end_pfn. 1674 * 1675 * It returns the start and end page frame of a node based on information 1676 * provided by memblock_set_node(). If called for a node 1677 * with no available memory, the start and end PFNs will be 0. 1678 */ 1679 void __init get_pfn_range_for_nid(unsigned int nid, 1680 unsigned long *start_pfn, unsigned long *end_pfn) 1681 { 1682 unsigned long this_start_pfn, this_end_pfn; 1683 int i; 1684 1685 *start_pfn = -1UL; 1686 *end_pfn = 0; 1687 1688 for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) { 1689 *start_pfn = min(*start_pfn, this_start_pfn); 1690 *end_pfn = max(*end_pfn, this_end_pfn); 1691 } 1692 1693 if (*start_pfn == -1UL) 1694 *start_pfn = 0; 1695 } 1696 1697 static void __init free_area_init_node(int nid) 1698 { 1699 pg_data_t *pgdat = NODE_DATA(nid); 1700 unsigned long start_pfn = 0; 1701 unsigned long end_pfn = 0; 1702 1703 /* pg_data_t should be reset to zero when it's allocated */ 1704 WARN_ON(pgdat->nr_zones || pgdat->kswapd_highest_zoneidx); 1705 1706 get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); 1707 1708 pgdat->node_id = nid; 1709 pgdat->node_start_pfn = start_pfn; 1710 pgdat->per_cpu_nodestats = NULL; 1711 1712 if (start_pfn != end_pfn) { 1713 pr_info("Initmem setup node %d [mem %#018Lx-%#018Lx]\n", nid, 1714 (u64)start_pfn << PAGE_SHIFT, 1715 end_pfn ? ((u64)end_pfn << PAGE_SHIFT) - 1 : 0); 1716 1717 calculate_node_totalpages(pgdat, start_pfn, end_pfn); 1718 } else { 1719 pr_info("Initmem setup node %d as memoryless\n", nid); 1720 1721 reset_memoryless_node_totalpages(pgdat); 1722 } 1723 1724 alloc_node_mem_map(pgdat); 1725 pgdat_set_deferred_range(pgdat); 1726 1727 free_area_init_core(pgdat); 1728 lru_gen_init_pgdat(pgdat); 1729 } 1730 1731 /* Any regular or high memory on that node ? */ 1732 static void __init check_for_memory(pg_data_t *pgdat) 1733 { 1734 enum zone_type zone_type; 1735 1736 for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) { 1737 struct zone *zone = &pgdat->node_zones[zone_type]; 1738 if (populated_zone(zone)) { 1739 if (IS_ENABLED(CONFIG_HIGHMEM)) 1740 node_set_state(pgdat->node_id, N_HIGH_MEMORY); 1741 if (zone_type <= ZONE_NORMAL) 1742 node_set_state(pgdat->node_id, N_NORMAL_MEMORY); 1743 break; 1744 } 1745 } 1746 } 1747 1748 #if MAX_NUMNODES > 1 1749 /* 1750 * Figure out the number of possible node ids. 1751 */ 1752 void __init setup_nr_node_ids(void) 1753 { 1754 unsigned int highest; 1755 1756 highest = find_last_bit(node_possible_map.bits, MAX_NUMNODES); 1757 nr_node_ids = highest + 1; 1758 } 1759 #endif 1760 1761 /* 1762 * Some architectures, e.g. ARC may have ZONE_HIGHMEM below ZONE_NORMAL. For 1763 * such cases we allow max_zone_pfn sorted in the descending order 1764 */ 1765 static bool arch_has_descending_max_zone_pfns(void) 1766 { 1767 return IS_ENABLED(CONFIG_ARC) && !IS_ENABLED(CONFIG_ARC_HAS_PAE40); 1768 } 1769 1770 /** 1771 * free_area_init - Initialise all pg_data_t and zone data 1772 * @max_zone_pfn: an array of max PFNs for each zone 1773 * 1774 * This will call free_area_init_node() for each active node in the system. 1775 * Using the page ranges provided by memblock_set_node(), the size of each 1776 * zone in each node and their holes is calculated. If the maximum PFN 1777 * between two adjacent zones match, it is assumed that the zone is empty. 1778 * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed 1779 * that arch_max_dma32_pfn has no pages. It is also assumed that a zone 1780 * starts where the previous one ended. For example, ZONE_DMA32 starts 1781 * at arch_max_dma_pfn. 1782 */ 1783 void __init free_area_init(unsigned long *max_zone_pfn) 1784 { 1785 unsigned long start_pfn, end_pfn; 1786 int i, nid, zone; 1787 bool descending; 1788 1789 /* Record where the zone boundaries are */ 1790 memset(arch_zone_lowest_possible_pfn, 0, 1791 sizeof(arch_zone_lowest_possible_pfn)); 1792 memset(arch_zone_highest_possible_pfn, 0, 1793 sizeof(arch_zone_highest_possible_pfn)); 1794 1795 start_pfn = PHYS_PFN(memblock_start_of_DRAM()); 1796 descending = arch_has_descending_max_zone_pfns(); 1797 1798 for (i = 0; i < MAX_NR_ZONES; i++) { 1799 if (descending) 1800 zone = MAX_NR_ZONES - i - 1; 1801 else 1802 zone = i; 1803 1804 if (zone == ZONE_MOVABLE) 1805 continue; 1806 1807 end_pfn = max(max_zone_pfn[zone], start_pfn); 1808 arch_zone_lowest_possible_pfn[zone] = start_pfn; 1809 arch_zone_highest_possible_pfn[zone] = end_pfn; 1810 1811 start_pfn = end_pfn; 1812 } 1813 1814 /* Find the PFNs that ZONE_MOVABLE begins at in each node */ 1815 memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn)); 1816 find_zone_movable_pfns_for_nodes(); 1817 1818 /* Print out the zone ranges */ 1819 pr_info("Zone ranges:\n"); 1820 for (i = 0; i < MAX_NR_ZONES; i++) { 1821 if (i == ZONE_MOVABLE) 1822 continue; 1823 pr_info(" %-8s ", zone_names[i]); 1824 if (arch_zone_lowest_possible_pfn[i] == 1825 arch_zone_highest_possible_pfn[i]) 1826 pr_cont("empty\n"); 1827 else 1828 pr_cont("[mem %#018Lx-%#018Lx]\n", 1829 (u64)arch_zone_lowest_possible_pfn[i] 1830 << PAGE_SHIFT, 1831 ((u64)arch_zone_highest_possible_pfn[i] 1832 << PAGE_SHIFT) - 1); 1833 } 1834 1835 /* Print out the PFNs ZONE_MOVABLE begins at in each node */ 1836 pr_info("Movable zone start for each node\n"); 1837 for (i = 0; i < MAX_NUMNODES; i++) { 1838 if (zone_movable_pfn[i]) 1839 pr_info(" Node %d: %#018Lx\n", i, 1840 (u64)zone_movable_pfn[i] << PAGE_SHIFT); 1841 } 1842 1843 /* 1844 * Print out the early node map, and initialize the 1845 * subsection-map relative to active online memory ranges to 1846 * enable future "sub-section" extensions of the memory map. 1847 */ 1848 pr_info("Early memory node ranges\n"); 1849 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 1850 pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid, 1851 (u64)start_pfn << PAGE_SHIFT, 1852 ((u64)end_pfn << PAGE_SHIFT) - 1); 1853 subsection_map_init(start_pfn, end_pfn - start_pfn); 1854 } 1855 1856 /* Initialise every node */ 1857 mminit_verify_pageflags_layout(); 1858 setup_nr_node_ids(); 1859 set_pageblock_order(); 1860 1861 for_each_node(nid) { 1862 pg_data_t *pgdat; 1863 1864 if (!node_online(nid)) { 1865 /* Allocator not initialized yet */ 1866 pgdat = arch_alloc_nodedata(nid); 1867 if (!pgdat) 1868 panic("Cannot allocate %zuB for node %d.\n", 1869 sizeof(*pgdat), nid); 1870 arch_refresh_nodedata(nid, pgdat); 1871 free_area_init_node(nid); 1872 1873 /* 1874 * We do not want to confuse userspace by sysfs 1875 * files/directories for node without any memory 1876 * attached to it, so this node is not marked as 1877 * N_MEMORY and not marked online so that no sysfs 1878 * hierarchy will be created via register_one_node for 1879 * it. The pgdat will get fully initialized by 1880 * hotadd_init_pgdat() when memory is hotplugged into 1881 * this node. 1882 */ 1883 continue; 1884 } 1885 1886 pgdat = NODE_DATA(nid); 1887 free_area_init_node(nid); 1888 1889 /* Any memory on that node */ 1890 if (pgdat->node_present_pages) 1891 node_set_state(nid, N_MEMORY); 1892 check_for_memory(pgdat); 1893 } 1894 1895 memmap_init(); 1896 1897 /* disable hash distribution for systems with a single node */ 1898 fixup_hashdist(); 1899 } 1900 1901 /** 1902 * node_map_pfn_alignment - determine the maximum internode alignment 1903 * 1904 * This function should be called after node map is populated and sorted. 1905 * It calculates the maximum power of two alignment which can distinguish 1906 * all the nodes. 1907 * 1908 * For example, if all nodes are 1GiB and aligned to 1GiB, the return value 1909 * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)). If the 1910 * nodes are shifted by 256MiB, 256MiB. Note that if only the last node is 1911 * shifted, 1GiB is enough and this function will indicate so. 1912 * 1913 * This is used to test whether pfn -> nid mapping of the chosen memory 1914 * model has fine enough granularity to avoid incorrect mapping for the 1915 * populated node map. 1916 * 1917 * Return: the determined alignment in pfn's. 0 if there is no alignment 1918 * requirement (single node). 1919 */ 1920 unsigned long __init node_map_pfn_alignment(void) 1921 { 1922 unsigned long accl_mask = 0, last_end = 0; 1923 unsigned long start, end, mask; 1924 int last_nid = NUMA_NO_NODE; 1925 int i, nid; 1926 1927 for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) { 1928 if (!start || last_nid < 0 || last_nid == nid) { 1929 last_nid = nid; 1930 last_end = end; 1931 continue; 1932 } 1933 1934 /* 1935 * Start with a mask granular enough to pin-point to the 1936 * start pfn and tick off bits one-by-one until it becomes 1937 * too coarse to separate the current node from the last. 1938 */ 1939 mask = ~((1 << __ffs(start)) - 1); 1940 while (mask && last_end <= (start & (mask << 1))) 1941 mask <<= 1; 1942 1943 /* accumulate all internode masks */ 1944 accl_mask |= mask; 1945 } 1946 1947 /* convert mask to number of pages */ 1948 return ~accl_mask + 1; 1949 } 1950 1951 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1952 static void __init deferred_free_range(unsigned long pfn, 1953 unsigned long nr_pages) 1954 { 1955 struct page *page; 1956 unsigned long i; 1957 1958 if (!nr_pages) 1959 return; 1960 1961 page = pfn_to_page(pfn); 1962 1963 /* Free a large naturally-aligned chunk if possible */ 1964 if (nr_pages == MAX_ORDER_NR_PAGES && IS_MAX_ORDER_ALIGNED(pfn)) { 1965 for (i = 0; i < nr_pages; i += pageblock_nr_pages) 1966 set_pageblock_migratetype(page + i, MIGRATE_MOVABLE); 1967 __free_pages_core(page, MAX_PAGE_ORDER); 1968 return; 1969 } 1970 1971 /* Accept chunks smaller than MAX_PAGE_ORDER upfront */ 1972 accept_memory(PFN_PHYS(pfn), PFN_PHYS(pfn + nr_pages)); 1973 1974 for (i = 0; i < nr_pages; i++, page++, pfn++) { 1975 if (pageblock_aligned(pfn)) 1976 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 1977 __free_pages_core(page, 0); 1978 } 1979 } 1980 1981 /* Completion tracking for deferred_init_memmap() threads */ 1982 static atomic_t pgdat_init_n_undone __initdata; 1983 static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp); 1984 1985 static inline void __init pgdat_init_report_one_done(void) 1986 { 1987 if (atomic_dec_and_test(&pgdat_init_n_undone)) 1988 complete(&pgdat_init_all_done_comp); 1989 } 1990 1991 /* 1992 * Returns true if page needs to be initialized or freed to buddy allocator. 1993 * 1994 * We check if a current MAX_PAGE_ORDER block is valid by only checking the 1995 * validity of the head pfn. 1996 */ 1997 static inline bool __init deferred_pfn_valid(unsigned long pfn) 1998 { 1999 if (IS_MAX_ORDER_ALIGNED(pfn) && !pfn_valid(pfn)) 2000 return false; 2001 return true; 2002 } 2003 2004 /* 2005 * Free pages to buddy allocator. Try to free aligned pages in 2006 * MAX_ORDER_NR_PAGES sizes. 2007 */ 2008 static void __init deferred_free_pages(unsigned long pfn, 2009 unsigned long end_pfn) 2010 { 2011 unsigned long nr_free = 0; 2012 2013 for (; pfn < end_pfn; pfn++) { 2014 if (!deferred_pfn_valid(pfn)) { 2015 deferred_free_range(pfn - nr_free, nr_free); 2016 nr_free = 0; 2017 } else if (IS_MAX_ORDER_ALIGNED(pfn)) { 2018 deferred_free_range(pfn - nr_free, nr_free); 2019 nr_free = 1; 2020 } else { 2021 nr_free++; 2022 } 2023 } 2024 /* Free the last block of pages to allocator */ 2025 deferred_free_range(pfn - nr_free, nr_free); 2026 } 2027 2028 /* 2029 * Initialize struct pages. We minimize pfn page lookups and scheduler checks 2030 * by performing it only once every MAX_ORDER_NR_PAGES. 2031 * Return number of pages initialized. 2032 */ 2033 static unsigned long __init deferred_init_pages(struct zone *zone, 2034 unsigned long pfn, 2035 unsigned long end_pfn) 2036 { 2037 int nid = zone_to_nid(zone); 2038 unsigned long nr_pages = 0; 2039 int zid = zone_idx(zone); 2040 struct page *page = NULL; 2041 2042 for (; pfn < end_pfn; pfn++) { 2043 if (!deferred_pfn_valid(pfn)) { 2044 page = NULL; 2045 continue; 2046 } else if (!page || IS_MAX_ORDER_ALIGNED(pfn)) { 2047 page = pfn_to_page(pfn); 2048 } else { 2049 page++; 2050 } 2051 __init_single_page(page, pfn, zid, nid); 2052 nr_pages++; 2053 } 2054 return (nr_pages); 2055 } 2056 2057 /* 2058 * This function is meant to pre-load the iterator for the zone init. 2059 * Specifically it walks through the ranges until we are caught up to the 2060 * first_init_pfn value and exits there. If we never encounter the value we 2061 * return false indicating there are no valid ranges left. 2062 */ 2063 static bool __init 2064 deferred_init_mem_pfn_range_in_zone(u64 *i, struct zone *zone, 2065 unsigned long *spfn, unsigned long *epfn, 2066 unsigned long first_init_pfn) 2067 { 2068 u64 j; 2069 2070 /* 2071 * Start out by walking through the ranges in this zone that have 2072 * already been initialized. We don't need to do anything with them 2073 * so we just need to flush them out of the system. 2074 */ 2075 for_each_free_mem_pfn_range_in_zone(j, zone, spfn, epfn) { 2076 if (*epfn <= first_init_pfn) 2077 continue; 2078 if (*spfn < first_init_pfn) 2079 *spfn = first_init_pfn; 2080 *i = j; 2081 return true; 2082 } 2083 2084 return false; 2085 } 2086 2087 /* 2088 * Initialize and free pages. We do it in two loops: first we initialize 2089 * struct page, then free to buddy allocator, because while we are 2090 * freeing pages we can access pages that are ahead (computing buddy 2091 * page in __free_one_page()). 2092 * 2093 * In order to try and keep some memory in the cache we have the loop 2094 * broken along max page order boundaries. This way we will not cause 2095 * any issues with the buddy page computation. 2096 */ 2097 static unsigned long __init 2098 deferred_init_maxorder(u64 *i, struct zone *zone, unsigned long *start_pfn, 2099 unsigned long *end_pfn) 2100 { 2101 unsigned long mo_pfn = ALIGN(*start_pfn + 1, MAX_ORDER_NR_PAGES); 2102 unsigned long spfn = *start_pfn, epfn = *end_pfn; 2103 unsigned long nr_pages = 0; 2104 u64 j = *i; 2105 2106 /* First we loop through and initialize the page values */ 2107 for_each_free_mem_pfn_range_in_zone_from(j, zone, start_pfn, end_pfn) { 2108 unsigned long t; 2109 2110 if (mo_pfn <= *start_pfn) 2111 break; 2112 2113 t = min(mo_pfn, *end_pfn); 2114 nr_pages += deferred_init_pages(zone, *start_pfn, t); 2115 2116 if (mo_pfn < *end_pfn) { 2117 *start_pfn = mo_pfn; 2118 break; 2119 } 2120 } 2121 2122 /* Reset values and now loop through freeing pages as needed */ 2123 swap(j, *i); 2124 2125 for_each_free_mem_pfn_range_in_zone_from(j, zone, &spfn, &epfn) { 2126 unsigned long t; 2127 2128 if (mo_pfn <= spfn) 2129 break; 2130 2131 t = min(mo_pfn, epfn); 2132 deferred_free_pages(spfn, t); 2133 2134 if (mo_pfn <= epfn) 2135 break; 2136 } 2137 2138 return nr_pages; 2139 } 2140 2141 static void __init 2142 deferred_init_memmap_chunk(unsigned long start_pfn, unsigned long end_pfn, 2143 void *arg) 2144 { 2145 unsigned long spfn, epfn; 2146 struct zone *zone = arg; 2147 u64 i; 2148 2149 deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, start_pfn); 2150 2151 /* 2152 * Initialize and free pages in MAX_PAGE_ORDER sized increments so that 2153 * we can avoid introducing any issues with the buddy allocator. 2154 */ 2155 while (spfn < end_pfn) { 2156 deferred_init_maxorder(&i, zone, &spfn, &epfn); 2157 cond_resched(); 2158 } 2159 } 2160 2161 /* An arch may override for more concurrency. */ 2162 __weak int __init 2163 deferred_page_init_max_threads(const struct cpumask *node_cpumask) 2164 { 2165 return 1; 2166 } 2167 2168 /* Initialise remaining memory on a node */ 2169 static int __init deferred_init_memmap(void *data) 2170 { 2171 pg_data_t *pgdat = data; 2172 const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id); 2173 unsigned long spfn = 0, epfn = 0; 2174 unsigned long first_init_pfn, flags; 2175 unsigned long start = jiffies; 2176 struct zone *zone; 2177 int zid, max_threads; 2178 u64 i; 2179 2180 /* Bind memory initialisation thread to a local node if possible */ 2181 if (!cpumask_empty(cpumask)) 2182 set_cpus_allowed_ptr(current, cpumask); 2183 2184 pgdat_resize_lock(pgdat, &flags); 2185 first_init_pfn = pgdat->first_deferred_pfn; 2186 if (first_init_pfn == ULONG_MAX) { 2187 pgdat_resize_unlock(pgdat, &flags); 2188 pgdat_init_report_one_done(); 2189 return 0; 2190 } 2191 2192 /* Sanity check boundaries */ 2193 BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn); 2194 BUG_ON(pgdat->first_deferred_pfn > pgdat_end_pfn(pgdat)); 2195 pgdat->first_deferred_pfn = ULONG_MAX; 2196 2197 /* 2198 * Once we unlock here, the zone cannot be grown anymore, thus if an 2199 * interrupt thread must allocate this early in boot, zone must be 2200 * pre-grown prior to start of deferred page initialization. 2201 */ 2202 pgdat_resize_unlock(pgdat, &flags); 2203 2204 /* Only the highest zone is deferred so find it */ 2205 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 2206 zone = pgdat->node_zones + zid; 2207 if (first_init_pfn < zone_end_pfn(zone)) 2208 break; 2209 } 2210 2211 /* If the zone is empty somebody else may have cleared out the zone */ 2212 if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, 2213 first_init_pfn)) 2214 goto zone_empty; 2215 2216 max_threads = deferred_page_init_max_threads(cpumask); 2217 2218 while (spfn < epfn) { 2219 unsigned long epfn_align = ALIGN(epfn, PAGES_PER_SECTION); 2220 struct padata_mt_job job = { 2221 .thread_fn = deferred_init_memmap_chunk, 2222 .fn_arg = zone, 2223 .start = spfn, 2224 .size = epfn_align - spfn, 2225 .align = PAGES_PER_SECTION, 2226 .min_chunk = PAGES_PER_SECTION, 2227 .max_threads = max_threads, 2228 }; 2229 2230 padata_do_multithreaded(&job); 2231 deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, 2232 epfn_align); 2233 } 2234 zone_empty: 2235 /* Sanity check that the next zone really is unpopulated */ 2236 WARN_ON(++zid < MAX_NR_ZONES && populated_zone(++zone)); 2237 2238 pr_info("node %d deferred pages initialised in %ums\n", 2239 pgdat->node_id, jiffies_to_msecs(jiffies - start)); 2240 2241 pgdat_init_report_one_done(); 2242 return 0; 2243 } 2244 2245 /* 2246 * If this zone has deferred pages, try to grow it by initializing enough 2247 * deferred pages to satisfy the allocation specified by order, rounded up to 2248 * the nearest PAGES_PER_SECTION boundary. So we're adding memory in increments 2249 * of SECTION_SIZE bytes by initializing struct pages in increments of 2250 * PAGES_PER_SECTION * sizeof(struct page) bytes. 2251 * 2252 * Return true when zone was grown, otherwise return false. We return true even 2253 * when we grow less than requested, to let the caller decide if there are 2254 * enough pages to satisfy the allocation. 2255 * 2256 * Note: We use noinline because this function is needed only during boot, and 2257 * it is called from a __ref function _deferred_grow_zone. This way we are 2258 * making sure that it is not inlined into permanent text section. 2259 */ 2260 bool __init deferred_grow_zone(struct zone *zone, unsigned int order) 2261 { 2262 unsigned long nr_pages_needed = ALIGN(1 << order, PAGES_PER_SECTION); 2263 pg_data_t *pgdat = zone->zone_pgdat; 2264 unsigned long first_deferred_pfn = pgdat->first_deferred_pfn; 2265 unsigned long spfn, epfn, flags; 2266 unsigned long nr_pages = 0; 2267 u64 i; 2268 2269 /* Only the last zone may have deferred pages */ 2270 if (zone_end_pfn(zone) != pgdat_end_pfn(pgdat)) 2271 return false; 2272 2273 pgdat_resize_lock(pgdat, &flags); 2274 2275 /* 2276 * If someone grew this zone while we were waiting for spinlock, return 2277 * true, as there might be enough pages already. 2278 */ 2279 if (first_deferred_pfn != pgdat->first_deferred_pfn) { 2280 pgdat_resize_unlock(pgdat, &flags); 2281 return true; 2282 } 2283 2284 /* If the zone is empty somebody else may have cleared out the zone */ 2285 if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, 2286 first_deferred_pfn)) { 2287 pgdat->first_deferred_pfn = ULONG_MAX; 2288 pgdat_resize_unlock(pgdat, &flags); 2289 /* Retry only once. */ 2290 return first_deferred_pfn != ULONG_MAX; 2291 } 2292 2293 /* 2294 * Initialize and free pages in MAX_PAGE_ORDER sized increments so 2295 * that we can avoid introducing any issues with the buddy 2296 * allocator. 2297 */ 2298 while (spfn < epfn) { 2299 /* update our first deferred PFN for this section */ 2300 first_deferred_pfn = spfn; 2301 2302 nr_pages += deferred_init_maxorder(&i, zone, &spfn, &epfn); 2303 touch_nmi_watchdog(); 2304 2305 /* We should only stop along section boundaries */ 2306 if ((first_deferred_pfn ^ spfn) < PAGES_PER_SECTION) 2307 continue; 2308 2309 /* If our quota has been met we can stop here */ 2310 if (nr_pages >= nr_pages_needed) 2311 break; 2312 } 2313 2314 pgdat->first_deferred_pfn = spfn; 2315 pgdat_resize_unlock(pgdat, &flags); 2316 2317 return nr_pages > 0; 2318 } 2319 2320 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ 2321 2322 #ifdef CONFIG_CMA 2323 void __init init_cma_reserved_pageblock(struct page *page) 2324 { 2325 unsigned i = pageblock_nr_pages; 2326 struct page *p = page; 2327 2328 do { 2329 __ClearPageReserved(p); 2330 set_page_count(p, 0); 2331 } while (++p, --i); 2332 2333 set_pageblock_migratetype(page, MIGRATE_CMA); 2334 set_page_refcounted(page); 2335 __free_pages(page, pageblock_order); 2336 2337 adjust_managed_page_count(page, pageblock_nr_pages); 2338 page_zone(page)->cma_pages += pageblock_nr_pages; 2339 } 2340 #endif 2341 2342 void set_zone_contiguous(struct zone *zone) 2343 { 2344 unsigned long block_start_pfn = zone->zone_start_pfn; 2345 unsigned long block_end_pfn; 2346 2347 block_end_pfn = pageblock_end_pfn(block_start_pfn); 2348 for (; block_start_pfn < zone_end_pfn(zone); 2349 block_start_pfn = block_end_pfn, 2350 block_end_pfn += pageblock_nr_pages) { 2351 2352 block_end_pfn = min(block_end_pfn, zone_end_pfn(zone)); 2353 2354 if (!__pageblock_pfn_to_page(block_start_pfn, 2355 block_end_pfn, zone)) 2356 return; 2357 cond_resched(); 2358 } 2359 2360 /* We confirm that there is no hole */ 2361 zone->contiguous = true; 2362 } 2363 2364 void __init page_alloc_init_late(void) 2365 { 2366 struct zone *zone; 2367 int nid; 2368 2369 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 2370 2371 /* There will be num_node_state(N_MEMORY) threads */ 2372 atomic_set(&pgdat_init_n_undone, num_node_state(N_MEMORY)); 2373 for_each_node_state(nid, N_MEMORY) { 2374 kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid); 2375 } 2376 2377 /* Block until all are initialised */ 2378 wait_for_completion(&pgdat_init_all_done_comp); 2379 2380 /* 2381 * We initialized the rest of the deferred pages. Permanently disable 2382 * on-demand struct page initialization. 2383 */ 2384 static_branch_disable(&deferred_pages); 2385 2386 /* Reinit limits that are based on free pages after the kernel is up */ 2387 files_maxfiles_init(); 2388 #endif 2389 2390 buffer_init(); 2391 2392 /* Discard memblock private memory */ 2393 memblock_discard(); 2394 2395 for_each_node_state(nid, N_MEMORY) 2396 shuffle_free_memory(NODE_DATA(nid)); 2397 2398 for_each_populated_zone(zone) 2399 set_zone_contiguous(zone); 2400 2401 /* Initialize page ext after all struct pages are initialized. */ 2402 if (deferred_struct_pages) 2403 page_ext_init(); 2404 2405 page_alloc_sysctl_init(); 2406 } 2407 2408 #ifndef __HAVE_ARCH_RESERVED_KERNEL_PAGES 2409 /* 2410 * Returns the number of pages that arch has reserved but 2411 * is not known to alloc_large_system_hash(). 2412 */ 2413 static unsigned long __init arch_reserved_kernel_pages(void) 2414 { 2415 return 0; 2416 } 2417 #endif 2418 2419 /* 2420 * Adaptive scale is meant to reduce sizes of hash tables on large memory 2421 * machines. As memory size is increased the scale is also increased but at 2422 * slower pace. Starting from ADAPT_SCALE_BASE (64G), every time memory 2423 * quadruples the scale is increased by one, which means the size of hash table 2424 * only doubles, instead of quadrupling as well. 2425 * Because 32-bit systems cannot have large physical memory, where this scaling 2426 * makes sense, it is disabled on such platforms. 2427 */ 2428 #if __BITS_PER_LONG > 32 2429 #define ADAPT_SCALE_BASE (64ul << 30) 2430 #define ADAPT_SCALE_SHIFT 2 2431 #define ADAPT_SCALE_NPAGES (ADAPT_SCALE_BASE >> PAGE_SHIFT) 2432 #endif 2433 2434 /* 2435 * allocate a large system hash table from bootmem 2436 * - it is assumed that the hash table must contain an exact power-of-2 2437 * quantity of entries 2438 * - limit is the number of hash buckets, not the total allocation size 2439 */ 2440 void *__init alloc_large_system_hash(const char *tablename, 2441 unsigned long bucketsize, 2442 unsigned long numentries, 2443 int scale, 2444 int flags, 2445 unsigned int *_hash_shift, 2446 unsigned int *_hash_mask, 2447 unsigned long low_limit, 2448 unsigned long high_limit) 2449 { 2450 unsigned long long max = high_limit; 2451 unsigned long log2qty, size; 2452 void *table; 2453 gfp_t gfp_flags; 2454 bool virt; 2455 bool huge; 2456 2457 /* allow the kernel cmdline to have a say */ 2458 if (!numentries) { 2459 /* round applicable memory size up to nearest megabyte */ 2460 numentries = nr_kernel_pages; 2461 numentries -= arch_reserved_kernel_pages(); 2462 2463 /* It isn't necessary when PAGE_SIZE >= 1MB */ 2464 if (PAGE_SIZE < SZ_1M) 2465 numentries = round_up(numentries, SZ_1M / PAGE_SIZE); 2466 2467 #if __BITS_PER_LONG > 32 2468 if (!high_limit) { 2469 unsigned long adapt; 2470 2471 for (adapt = ADAPT_SCALE_NPAGES; adapt < numentries; 2472 adapt <<= ADAPT_SCALE_SHIFT) 2473 scale++; 2474 } 2475 #endif 2476 2477 /* limit to 1 bucket per 2^scale bytes of low memory */ 2478 if (scale > PAGE_SHIFT) 2479 numentries >>= (scale - PAGE_SHIFT); 2480 else 2481 numentries <<= (PAGE_SHIFT - scale); 2482 2483 if (unlikely((numentries * bucketsize) < PAGE_SIZE)) 2484 numentries = PAGE_SIZE / bucketsize; 2485 } 2486 numentries = roundup_pow_of_two(numentries); 2487 2488 /* limit allocation size to 1/16 total memory by default */ 2489 if (max == 0) { 2490 max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4; 2491 do_div(max, bucketsize); 2492 } 2493 max = min(max, 0x80000000ULL); 2494 2495 if (numentries < low_limit) 2496 numentries = low_limit; 2497 if (numentries > max) 2498 numentries = max; 2499 2500 log2qty = ilog2(numentries); 2501 2502 gfp_flags = (flags & HASH_ZERO) ? GFP_ATOMIC | __GFP_ZERO : GFP_ATOMIC; 2503 do { 2504 virt = false; 2505 size = bucketsize << log2qty; 2506 if (flags & HASH_EARLY) { 2507 if (flags & HASH_ZERO) 2508 table = memblock_alloc(size, SMP_CACHE_BYTES); 2509 else 2510 table = memblock_alloc_raw(size, 2511 SMP_CACHE_BYTES); 2512 } else if (get_order(size) > MAX_PAGE_ORDER || hashdist) { 2513 table = vmalloc_huge(size, gfp_flags); 2514 virt = true; 2515 if (table) 2516 huge = is_vm_area_hugepages(table); 2517 } else { 2518 /* 2519 * If bucketsize is not a power-of-two, we may free 2520 * some pages at the end of hash table which 2521 * alloc_pages_exact() automatically does 2522 */ 2523 table = alloc_pages_exact(size, gfp_flags); 2524 kmemleak_alloc(table, size, 1, gfp_flags); 2525 } 2526 } while (!table && size > PAGE_SIZE && --log2qty); 2527 2528 if (!table) 2529 panic("Failed to allocate %s hash table\n", tablename); 2530 2531 pr_info("%s hash table entries: %ld (order: %d, %lu bytes, %s)\n", 2532 tablename, 1UL << log2qty, ilog2(size) - PAGE_SHIFT, size, 2533 virt ? (huge ? "vmalloc hugepage" : "vmalloc") : "linear"); 2534 2535 if (_hash_shift) 2536 *_hash_shift = log2qty; 2537 if (_hash_mask) 2538 *_hash_mask = (1 << log2qty) - 1; 2539 2540 return table; 2541 } 2542 2543 /** 2544 * set_dma_reserve - set the specified number of pages reserved in the first zone 2545 * @new_dma_reserve: The number of pages to mark reserved 2546 * 2547 * The per-cpu batchsize and zone watermarks are determined by managed_pages. 2548 * In the DMA zone, a significant percentage may be consumed by kernel image 2549 * and other unfreeable allocations which can skew the watermarks badly. This 2550 * function may optionally be used to account for unfreeable pages in the 2551 * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and 2552 * smaller per-cpu batchsize. 2553 */ 2554 void __init set_dma_reserve(unsigned long new_dma_reserve) 2555 { 2556 dma_reserve = new_dma_reserve; 2557 } 2558 2559 void __init memblock_free_pages(struct page *page, unsigned long pfn, 2560 unsigned int order) 2561 { 2562 2563 if (IS_ENABLED(CONFIG_DEFERRED_STRUCT_PAGE_INIT)) { 2564 int nid = early_pfn_to_nid(pfn); 2565 2566 if (!early_page_initialised(pfn, nid)) 2567 return; 2568 } 2569 2570 if (!kmsan_memblock_free_pages(page, order)) { 2571 /* KMSAN will take care of these pages. */ 2572 return; 2573 } 2574 __free_pages_core(page, order); 2575 } 2576 2577 DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, init_on_alloc); 2578 EXPORT_SYMBOL(init_on_alloc); 2579 2580 DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_FREE_DEFAULT_ON, init_on_free); 2581 EXPORT_SYMBOL(init_on_free); 2582 2583 static bool _init_on_alloc_enabled_early __read_mostly 2584 = IS_ENABLED(CONFIG_INIT_ON_ALLOC_DEFAULT_ON); 2585 static int __init early_init_on_alloc(char *buf) 2586 { 2587 2588 return kstrtobool(buf, &_init_on_alloc_enabled_early); 2589 } 2590 early_param("init_on_alloc", early_init_on_alloc); 2591 2592 static bool _init_on_free_enabled_early __read_mostly 2593 = IS_ENABLED(CONFIG_INIT_ON_FREE_DEFAULT_ON); 2594 static int __init early_init_on_free(char *buf) 2595 { 2596 return kstrtobool(buf, &_init_on_free_enabled_early); 2597 } 2598 early_param("init_on_free", early_init_on_free); 2599 2600 DEFINE_STATIC_KEY_MAYBE(CONFIG_DEBUG_VM, check_pages_enabled); 2601 2602 /* 2603 * Enable static keys related to various memory debugging and hardening options. 2604 * Some override others, and depend on early params that are evaluated in the 2605 * order of appearance. So we need to first gather the full picture of what was 2606 * enabled, and then make decisions. 2607 */ 2608 static void __init mem_debugging_and_hardening_init(void) 2609 { 2610 bool page_poisoning_requested = false; 2611 bool want_check_pages = false; 2612 2613 #ifdef CONFIG_PAGE_POISONING 2614 /* 2615 * Page poisoning is debug page alloc for some arches. If 2616 * either of those options are enabled, enable poisoning. 2617 */ 2618 if (page_poisoning_enabled() || 2619 (!IS_ENABLED(CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC) && 2620 debug_pagealloc_enabled())) { 2621 static_branch_enable(&_page_poisoning_enabled); 2622 page_poisoning_requested = true; 2623 want_check_pages = true; 2624 } 2625 #endif 2626 2627 if ((_init_on_alloc_enabled_early || _init_on_free_enabled_early) && 2628 page_poisoning_requested) { 2629 pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, " 2630 "will take precedence over init_on_alloc and init_on_free\n"); 2631 _init_on_alloc_enabled_early = false; 2632 _init_on_free_enabled_early = false; 2633 } 2634 2635 if (_init_on_alloc_enabled_early) { 2636 want_check_pages = true; 2637 static_branch_enable(&init_on_alloc); 2638 } else { 2639 static_branch_disable(&init_on_alloc); 2640 } 2641 2642 if (_init_on_free_enabled_early) { 2643 want_check_pages = true; 2644 static_branch_enable(&init_on_free); 2645 } else { 2646 static_branch_disable(&init_on_free); 2647 } 2648 2649 if (IS_ENABLED(CONFIG_KMSAN) && 2650 (_init_on_alloc_enabled_early || _init_on_free_enabled_early)) 2651 pr_info("mem auto-init: please make sure init_on_alloc and init_on_free are disabled when running KMSAN\n"); 2652 2653 #ifdef CONFIG_DEBUG_PAGEALLOC 2654 if (debug_pagealloc_enabled()) { 2655 want_check_pages = true; 2656 static_branch_enable(&_debug_pagealloc_enabled); 2657 2658 if (debug_guardpage_minorder()) 2659 static_branch_enable(&_debug_guardpage_enabled); 2660 } 2661 #endif 2662 2663 /* 2664 * Any page debugging or hardening option also enables sanity checking 2665 * of struct pages being allocated or freed. With CONFIG_DEBUG_VM it's 2666 * enabled already. 2667 */ 2668 if (!IS_ENABLED(CONFIG_DEBUG_VM) && want_check_pages) 2669 static_branch_enable(&check_pages_enabled); 2670 } 2671 2672 /* Report memory auto-initialization states for this boot. */ 2673 static void __init report_meminit(void) 2674 { 2675 const char *stack; 2676 2677 if (IS_ENABLED(CONFIG_INIT_STACK_ALL_PATTERN)) 2678 stack = "all(pattern)"; 2679 else if (IS_ENABLED(CONFIG_INIT_STACK_ALL_ZERO)) 2680 stack = "all(zero)"; 2681 else if (IS_ENABLED(CONFIG_GCC_PLUGIN_STRUCTLEAK_BYREF_ALL)) 2682 stack = "byref_all(zero)"; 2683 else if (IS_ENABLED(CONFIG_GCC_PLUGIN_STRUCTLEAK_BYREF)) 2684 stack = "byref(zero)"; 2685 else if (IS_ENABLED(CONFIG_GCC_PLUGIN_STRUCTLEAK_USER)) 2686 stack = "__user(zero)"; 2687 else 2688 stack = "off"; 2689 2690 pr_info("mem auto-init: stack:%s, heap alloc:%s, heap free:%s\n", 2691 stack, want_init_on_alloc(GFP_KERNEL) ? "on" : "off", 2692 want_init_on_free() ? "on" : "off"); 2693 if (want_init_on_free()) 2694 pr_info("mem auto-init: clearing system memory may take some time...\n"); 2695 } 2696 2697 static void __init mem_init_print_info(void) 2698 { 2699 unsigned long physpages, codesize, datasize, rosize, bss_size; 2700 unsigned long init_code_size, init_data_size; 2701 2702 physpages = get_num_physpages(); 2703 codesize = _etext - _stext; 2704 datasize = _edata - _sdata; 2705 rosize = __end_rodata - __start_rodata; 2706 bss_size = __bss_stop - __bss_start; 2707 init_data_size = __init_end - __init_begin; 2708 init_code_size = _einittext - _sinittext; 2709 2710 /* 2711 * Detect special cases and adjust section sizes accordingly: 2712 * 1) .init.* may be embedded into .data sections 2713 * 2) .init.text.* may be out of [__init_begin, __init_end], 2714 * please refer to arch/tile/kernel/vmlinux.lds.S. 2715 * 3) .rodata.* may be embedded into .text or .data sections. 2716 */ 2717 #define adj_init_size(start, end, size, pos, adj) \ 2718 do { \ 2719 if (&start[0] <= &pos[0] && &pos[0] < &end[0] && size > adj) \ 2720 size -= adj; \ 2721 } while (0) 2722 2723 adj_init_size(__init_begin, __init_end, init_data_size, 2724 _sinittext, init_code_size); 2725 adj_init_size(_stext, _etext, codesize, _sinittext, init_code_size); 2726 adj_init_size(_sdata, _edata, datasize, __init_begin, init_data_size); 2727 adj_init_size(_stext, _etext, codesize, __start_rodata, rosize); 2728 adj_init_size(_sdata, _edata, datasize, __start_rodata, rosize); 2729 2730 #undef adj_init_size 2731 2732 pr_info("Memory: %luK/%luK available (%luK kernel code, %luK rwdata, %luK rodata, %luK init, %luK bss, %luK reserved, %luK cma-reserved" 2733 #ifdef CONFIG_HIGHMEM 2734 ", %luK highmem" 2735 #endif 2736 ")\n", 2737 K(nr_free_pages()), K(physpages), 2738 codesize / SZ_1K, datasize / SZ_1K, rosize / SZ_1K, 2739 (init_data_size + init_code_size) / SZ_1K, bss_size / SZ_1K, 2740 K(physpages - totalram_pages() - totalcma_pages), 2741 K(totalcma_pages) 2742 #ifdef CONFIG_HIGHMEM 2743 , K(totalhigh_pages()) 2744 #endif 2745 ); 2746 } 2747 2748 /* 2749 * Set up kernel memory allocators 2750 */ 2751 void __init mm_core_init(void) 2752 { 2753 /* Initializations relying on SMP setup */ 2754 build_all_zonelists(NULL); 2755 page_alloc_init_cpuhp(); 2756 2757 /* 2758 * page_ext requires contiguous pages, 2759 * bigger than MAX_PAGE_ORDER unless SPARSEMEM. 2760 */ 2761 page_ext_init_flatmem(); 2762 mem_debugging_and_hardening_init(); 2763 kfence_alloc_pool_and_metadata(); 2764 report_meminit(); 2765 kmsan_init_shadow(); 2766 stack_depot_early_init(); 2767 mem_init(); 2768 mem_init_print_info(); 2769 kmem_cache_init(); 2770 /* 2771 * page_owner must be initialized after buddy is ready, and also after 2772 * slab is ready so that stack_depot_init() works properly 2773 */ 2774 page_ext_init_flatmem_late(); 2775 kmemleak_init(); 2776 ptlock_cache_init(); 2777 pgtable_cache_init(); 2778 debug_objects_mem_init(); 2779 vmalloc_init(); 2780 /* If no deferred init page_ext now, as vmap is fully initialized */ 2781 if (!deferred_struct_pages) 2782 page_ext_init(); 2783 /* Should be run before the first non-init thread is created */ 2784 init_espfix_bsp(); 2785 /* Should be run after espfix64 is set up. */ 2786 pti_init(); 2787 kmsan_init_runtime(); 2788 mm_cache_init(); 2789 } 2790