1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/mm/page_alloc.c 4 * 5 * Manages the free list, the system allocates free pages here. 6 * Note that kmalloc() lives in slab.c 7 * 8 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 9 * Swap reorganised 29.12.95, Stephen Tweedie 10 * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 11 * Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999 12 * Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999 13 * Zone balancing, Kanoj Sarcar, SGI, Jan 2000 14 * Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002 15 * (lots of bits borrowed from Ingo Molnar & Andrew Morton) 16 */ 17 18 #include <linux/stddef.h> 19 #include <linux/mm.h> 20 #include <linux/highmem.h> 21 #include <linux/swap.h> 22 #include <linux/swapops.h> 23 #include <linux/interrupt.h> 24 #include <linux/pagemap.h> 25 #include <linux/jiffies.h> 26 #include <linux/memblock.h> 27 #include <linux/compiler.h> 28 #include <linux/kernel.h> 29 #include <linux/kasan.h> 30 #include <linux/module.h> 31 #include <linux/suspend.h> 32 #include <linux/pagevec.h> 33 #include <linux/blkdev.h> 34 #include <linux/slab.h> 35 #include <linux/ratelimit.h> 36 #include <linux/oom.h> 37 #include <linux/topology.h> 38 #include <linux/sysctl.h> 39 #include <linux/cpu.h> 40 #include <linux/cpuset.h> 41 #include <linux/memory_hotplug.h> 42 #include <linux/nodemask.h> 43 #include <linux/vmalloc.h> 44 #include <linux/vmstat.h> 45 #include <linux/mempolicy.h> 46 #include <linux/memremap.h> 47 #include <linux/stop_machine.h> 48 #include <linux/random.h> 49 #include <linux/sort.h> 50 #include <linux/pfn.h> 51 #include <linux/backing-dev.h> 52 #include <linux/fault-inject.h> 53 #include <linux/page-isolation.h> 54 #include <linux/debugobjects.h> 55 #include <linux/kmemleak.h> 56 #include <linux/compaction.h> 57 #include <trace/events/kmem.h> 58 #include <trace/events/oom.h> 59 #include <linux/prefetch.h> 60 #include <linux/mm_inline.h> 61 #include <linux/mmu_notifier.h> 62 #include <linux/migrate.h> 63 #include <linux/hugetlb.h> 64 #include <linux/sched/rt.h> 65 #include <linux/sched/mm.h> 66 #include <linux/page_owner.h> 67 #include <linux/page_table_check.h> 68 #include <linux/kthread.h> 69 #include <linux/memcontrol.h> 70 #include <linux/ftrace.h> 71 #include <linux/lockdep.h> 72 #include <linux/nmi.h> 73 #include <linux/psi.h> 74 #include <linux/padata.h> 75 #include <linux/khugepaged.h> 76 #include <linux/buffer_head.h> 77 #include <linux/delayacct.h> 78 #include <asm/sections.h> 79 #include <asm/tlbflush.h> 80 #include <asm/div64.h> 81 #include "internal.h" 82 #include "shuffle.h" 83 #include "page_reporting.h" 84 85 /* Free Page Internal flags: for internal, non-pcp variants of free_pages(). */ 86 typedef int __bitwise fpi_t; 87 88 /* No special request */ 89 #define FPI_NONE ((__force fpi_t)0) 90 91 /* 92 * Skip free page reporting notification for the (possibly merged) page. 93 * This does not hinder free page reporting from grabbing the page, 94 * reporting it and marking it "reported" - it only skips notifying 95 * the free page reporting infrastructure about a newly freed page. For 96 * example, used when temporarily pulling a page from a freelist and 97 * putting it back unmodified. 98 */ 99 #define FPI_SKIP_REPORT_NOTIFY ((__force fpi_t)BIT(0)) 100 101 /* 102 * Place the (possibly merged) page to the tail of the freelist. Will ignore 103 * page shuffling (relevant code - e.g., memory onlining - is expected to 104 * shuffle the whole zone). 105 * 106 * Note: No code should rely on this flag for correctness - it's purely 107 * to allow for optimizations when handing back either fresh pages 108 * (memory onlining) or untouched pages (page isolation, free page 109 * reporting). 110 */ 111 #define FPI_TO_TAIL ((__force fpi_t)BIT(1)) 112 113 /* 114 * Don't poison memory with KASAN (only for the tag-based modes). 115 * During boot, all non-reserved memblock memory is exposed to page_alloc. 116 * Poisoning all that memory lengthens boot time, especially on systems with 117 * large amount of RAM. This flag is used to skip that poisoning. 118 * This is only done for the tag-based KASAN modes, as those are able to 119 * detect memory corruptions with the memory tags assigned by default. 120 * All memory allocated normally after boot gets poisoned as usual. 121 */ 122 #define FPI_SKIP_KASAN_POISON ((__force fpi_t)BIT(2)) 123 124 /* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */ 125 static DEFINE_MUTEX(pcp_batch_high_lock); 126 #define MIN_PERCPU_PAGELIST_HIGH_FRACTION (8) 127 128 struct pagesets { 129 local_lock_t lock; 130 }; 131 static DEFINE_PER_CPU(struct pagesets, pagesets) = { 132 .lock = INIT_LOCAL_LOCK(lock), 133 }; 134 135 #ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID 136 DEFINE_PER_CPU(int, numa_node); 137 EXPORT_PER_CPU_SYMBOL(numa_node); 138 #endif 139 140 DEFINE_STATIC_KEY_TRUE(vm_numa_stat_key); 141 142 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 143 /* 144 * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly. 145 * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined. 146 * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem() 147 * defined in <linux/topology.h>. 148 */ 149 DEFINE_PER_CPU(int, _numa_mem_); /* Kernel "local memory" node */ 150 EXPORT_PER_CPU_SYMBOL(_numa_mem_); 151 #endif 152 153 /* work_structs for global per-cpu drains */ 154 struct pcpu_drain { 155 struct zone *zone; 156 struct work_struct work; 157 }; 158 static DEFINE_MUTEX(pcpu_drain_mutex); 159 static DEFINE_PER_CPU(struct pcpu_drain, pcpu_drain); 160 161 #ifdef CONFIG_GCC_PLUGIN_LATENT_ENTROPY 162 volatile unsigned long latent_entropy __latent_entropy; 163 EXPORT_SYMBOL(latent_entropy); 164 #endif 165 166 /* 167 * Array of node states. 168 */ 169 nodemask_t node_states[NR_NODE_STATES] __read_mostly = { 170 [N_POSSIBLE] = NODE_MASK_ALL, 171 [N_ONLINE] = { { [0] = 1UL } }, 172 #ifndef CONFIG_NUMA 173 [N_NORMAL_MEMORY] = { { [0] = 1UL } }, 174 #ifdef CONFIG_HIGHMEM 175 [N_HIGH_MEMORY] = { { [0] = 1UL } }, 176 #endif 177 [N_MEMORY] = { { [0] = 1UL } }, 178 [N_CPU] = { { [0] = 1UL } }, 179 #endif /* NUMA */ 180 }; 181 EXPORT_SYMBOL(node_states); 182 183 atomic_long_t _totalram_pages __read_mostly; 184 EXPORT_SYMBOL(_totalram_pages); 185 unsigned long totalreserve_pages __read_mostly; 186 unsigned long totalcma_pages __read_mostly; 187 188 int percpu_pagelist_high_fraction; 189 gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK; 190 DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, init_on_alloc); 191 EXPORT_SYMBOL(init_on_alloc); 192 193 DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_FREE_DEFAULT_ON, init_on_free); 194 EXPORT_SYMBOL(init_on_free); 195 196 static bool _init_on_alloc_enabled_early __read_mostly 197 = IS_ENABLED(CONFIG_INIT_ON_ALLOC_DEFAULT_ON); 198 static int __init early_init_on_alloc(char *buf) 199 { 200 201 return kstrtobool(buf, &_init_on_alloc_enabled_early); 202 } 203 early_param("init_on_alloc", early_init_on_alloc); 204 205 static bool _init_on_free_enabled_early __read_mostly 206 = IS_ENABLED(CONFIG_INIT_ON_FREE_DEFAULT_ON); 207 static int __init early_init_on_free(char *buf) 208 { 209 return kstrtobool(buf, &_init_on_free_enabled_early); 210 } 211 early_param("init_on_free", early_init_on_free); 212 213 /* 214 * A cached value of the page's pageblock's migratetype, used when the page is 215 * put on a pcplist. Used to avoid the pageblock migratetype lookup when 216 * freeing from pcplists in most cases, at the cost of possibly becoming stale. 217 * Also the migratetype set in the page does not necessarily match the pcplist 218 * index, e.g. page might have MIGRATE_CMA set but be on a pcplist with any 219 * other index - this ensures that it will be put on the correct CMA freelist. 220 */ 221 static inline int get_pcppage_migratetype(struct page *page) 222 { 223 return page->index; 224 } 225 226 static inline void set_pcppage_migratetype(struct page *page, int migratetype) 227 { 228 page->index = migratetype; 229 } 230 231 #ifdef CONFIG_PM_SLEEP 232 /* 233 * The following functions are used by the suspend/hibernate code to temporarily 234 * change gfp_allowed_mask in order to avoid using I/O during memory allocations 235 * while devices are suspended. To avoid races with the suspend/hibernate code, 236 * they should always be called with system_transition_mutex held 237 * (gfp_allowed_mask also should only be modified with system_transition_mutex 238 * held, unless the suspend/hibernate code is guaranteed not to run in parallel 239 * with that modification). 240 */ 241 242 static gfp_t saved_gfp_mask; 243 244 void pm_restore_gfp_mask(void) 245 { 246 WARN_ON(!mutex_is_locked(&system_transition_mutex)); 247 if (saved_gfp_mask) { 248 gfp_allowed_mask = saved_gfp_mask; 249 saved_gfp_mask = 0; 250 } 251 } 252 253 void pm_restrict_gfp_mask(void) 254 { 255 WARN_ON(!mutex_is_locked(&system_transition_mutex)); 256 WARN_ON(saved_gfp_mask); 257 saved_gfp_mask = gfp_allowed_mask; 258 gfp_allowed_mask &= ~(__GFP_IO | __GFP_FS); 259 } 260 261 bool pm_suspended_storage(void) 262 { 263 if ((gfp_allowed_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS)) 264 return false; 265 return true; 266 } 267 #endif /* CONFIG_PM_SLEEP */ 268 269 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE 270 unsigned int pageblock_order __read_mostly; 271 #endif 272 273 static void __free_pages_ok(struct page *page, unsigned int order, 274 fpi_t fpi_flags); 275 276 /* 277 * results with 256, 32 in the lowmem_reserve sysctl: 278 * 1G machine -> (16M dma, 800M-16M normal, 1G-800M high) 279 * 1G machine -> (16M dma, 784M normal, 224M high) 280 * NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA 281 * HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL 282 * HIGHMEM allocation will leave (224M+784M)/256 of ram reserved in ZONE_DMA 283 * 284 * TBD: should special case ZONE_DMA32 machines here - in those we normally 285 * don't need any ZONE_NORMAL reservation 286 */ 287 int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES] = { 288 #ifdef CONFIG_ZONE_DMA 289 [ZONE_DMA] = 256, 290 #endif 291 #ifdef CONFIG_ZONE_DMA32 292 [ZONE_DMA32] = 256, 293 #endif 294 [ZONE_NORMAL] = 32, 295 #ifdef CONFIG_HIGHMEM 296 [ZONE_HIGHMEM] = 0, 297 #endif 298 [ZONE_MOVABLE] = 0, 299 }; 300 301 static char * const zone_names[MAX_NR_ZONES] = { 302 #ifdef CONFIG_ZONE_DMA 303 "DMA", 304 #endif 305 #ifdef CONFIG_ZONE_DMA32 306 "DMA32", 307 #endif 308 "Normal", 309 #ifdef CONFIG_HIGHMEM 310 "HighMem", 311 #endif 312 "Movable", 313 #ifdef CONFIG_ZONE_DEVICE 314 "Device", 315 #endif 316 }; 317 318 const char * const migratetype_names[MIGRATE_TYPES] = { 319 "Unmovable", 320 "Movable", 321 "Reclaimable", 322 "HighAtomic", 323 #ifdef CONFIG_CMA 324 "CMA", 325 #endif 326 #ifdef CONFIG_MEMORY_ISOLATION 327 "Isolate", 328 #endif 329 }; 330 331 compound_page_dtor * const compound_page_dtors[NR_COMPOUND_DTORS] = { 332 [NULL_COMPOUND_DTOR] = NULL, 333 [COMPOUND_PAGE_DTOR] = free_compound_page, 334 #ifdef CONFIG_HUGETLB_PAGE 335 [HUGETLB_PAGE_DTOR] = free_huge_page, 336 #endif 337 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 338 [TRANSHUGE_PAGE_DTOR] = free_transhuge_page, 339 #endif 340 }; 341 342 int min_free_kbytes = 1024; 343 int user_min_free_kbytes = -1; 344 int watermark_boost_factor __read_mostly = 15000; 345 int watermark_scale_factor = 10; 346 347 static unsigned long nr_kernel_pages __initdata; 348 static unsigned long nr_all_pages __initdata; 349 static unsigned long dma_reserve __initdata; 350 351 static unsigned long arch_zone_lowest_possible_pfn[MAX_NR_ZONES] __initdata; 352 static unsigned long arch_zone_highest_possible_pfn[MAX_NR_ZONES] __initdata; 353 static unsigned long required_kernelcore __initdata; 354 static unsigned long required_kernelcore_percent __initdata; 355 static unsigned long required_movablecore __initdata; 356 static unsigned long required_movablecore_percent __initdata; 357 static unsigned long zone_movable_pfn[MAX_NUMNODES] __initdata; 358 static bool mirrored_kernelcore __meminitdata; 359 360 /* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */ 361 int movable_zone; 362 EXPORT_SYMBOL(movable_zone); 363 364 #if MAX_NUMNODES > 1 365 unsigned int nr_node_ids __read_mostly = MAX_NUMNODES; 366 unsigned int nr_online_nodes __read_mostly = 1; 367 EXPORT_SYMBOL(nr_node_ids); 368 EXPORT_SYMBOL(nr_online_nodes); 369 #endif 370 371 int page_group_by_mobility_disabled __read_mostly; 372 373 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 374 /* 375 * During boot we initialize deferred pages on-demand, as needed, but once 376 * page_alloc_init_late() has finished, the deferred pages are all initialized, 377 * and we can permanently disable that path. 378 */ 379 static DEFINE_STATIC_KEY_TRUE(deferred_pages); 380 381 static inline bool deferred_pages_enabled(void) 382 { 383 return static_branch_unlikely(&deferred_pages); 384 } 385 386 /* Returns true if the struct page for the pfn is uninitialised */ 387 static inline bool __meminit early_page_uninitialised(unsigned long pfn) 388 { 389 int nid = early_pfn_to_nid(pfn); 390 391 if (node_online(nid) && pfn >= NODE_DATA(nid)->first_deferred_pfn) 392 return true; 393 394 return false; 395 } 396 397 /* 398 * Returns true when the remaining initialisation should be deferred until 399 * later in the boot cycle when it can be parallelised. 400 */ 401 static bool __meminit 402 defer_init(int nid, unsigned long pfn, unsigned long end_pfn) 403 { 404 static unsigned long prev_end_pfn, nr_initialised; 405 406 /* 407 * prev_end_pfn static that contains the end of previous zone 408 * No need to protect because called very early in boot before smp_init. 409 */ 410 if (prev_end_pfn != end_pfn) { 411 prev_end_pfn = end_pfn; 412 nr_initialised = 0; 413 } 414 415 /* Always populate low zones for address-constrained allocations */ 416 if (end_pfn < pgdat_end_pfn(NODE_DATA(nid))) 417 return false; 418 419 if (NODE_DATA(nid)->first_deferred_pfn != ULONG_MAX) 420 return true; 421 /* 422 * We start only with one section of pages, more pages are added as 423 * needed until the rest of deferred pages are initialized. 424 */ 425 nr_initialised++; 426 if ((nr_initialised > PAGES_PER_SECTION) && 427 (pfn & (PAGES_PER_SECTION - 1)) == 0) { 428 NODE_DATA(nid)->first_deferred_pfn = pfn; 429 return true; 430 } 431 return false; 432 } 433 #else 434 static inline bool deferred_pages_enabled(void) 435 { 436 return false; 437 } 438 439 static inline bool early_page_uninitialised(unsigned long pfn) 440 { 441 return false; 442 } 443 444 static inline bool defer_init(int nid, unsigned long pfn, unsigned long end_pfn) 445 { 446 return false; 447 } 448 #endif 449 450 /* Return a pointer to the bitmap storing bits affecting a block of pages */ 451 static inline unsigned long *get_pageblock_bitmap(const struct page *page, 452 unsigned long pfn) 453 { 454 #ifdef CONFIG_SPARSEMEM 455 return section_to_usemap(__pfn_to_section(pfn)); 456 #else 457 return page_zone(page)->pageblock_flags; 458 #endif /* CONFIG_SPARSEMEM */ 459 } 460 461 static inline int pfn_to_bitidx(const struct page *page, unsigned long pfn) 462 { 463 #ifdef CONFIG_SPARSEMEM 464 pfn &= (PAGES_PER_SECTION-1); 465 #else 466 pfn = pfn - round_down(page_zone(page)->zone_start_pfn, pageblock_nr_pages); 467 #endif /* CONFIG_SPARSEMEM */ 468 return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS; 469 } 470 471 static __always_inline 472 unsigned long __get_pfnblock_flags_mask(const struct page *page, 473 unsigned long pfn, 474 unsigned long mask) 475 { 476 unsigned long *bitmap; 477 unsigned long bitidx, word_bitidx; 478 unsigned long word; 479 480 bitmap = get_pageblock_bitmap(page, pfn); 481 bitidx = pfn_to_bitidx(page, pfn); 482 word_bitidx = bitidx / BITS_PER_LONG; 483 bitidx &= (BITS_PER_LONG-1); 484 485 word = bitmap[word_bitidx]; 486 return (word >> bitidx) & mask; 487 } 488 489 /** 490 * get_pfnblock_flags_mask - Return the requested group of flags for the pageblock_nr_pages block of pages 491 * @page: The page within the block of interest 492 * @pfn: The target page frame number 493 * @mask: mask of bits that the caller is interested in 494 * 495 * Return: pageblock_bits flags 496 */ 497 unsigned long get_pfnblock_flags_mask(const struct page *page, 498 unsigned long pfn, unsigned long mask) 499 { 500 return __get_pfnblock_flags_mask(page, pfn, mask); 501 } 502 503 static __always_inline int get_pfnblock_migratetype(const struct page *page, 504 unsigned long pfn) 505 { 506 return __get_pfnblock_flags_mask(page, pfn, MIGRATETYPE_MASK); 507 } 508 509 /** 510 * set_pfnblock_flags_mask - Set the requested group of flags for a pageblock_nr_pages block of pages 511 * @page: The page within the block of interest 512 * @flags: The flags to set 513 * @pfn: The target page frame number 514 * @mask: mask of bits that the caller is interested in 515 */ 516 void set_pfnblock_flags_mask(struct page *page, unsigned long flags, 517 unsigned long pfn, 518 unsigned long mask) 519 { 520 unsigned long *bitmap; 521 unsigned long bitidx, word_bitidx; 522 unsigned long old_word, word; 523 524 BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4); 525 BUILD_BUG_ON(MIGRATE_TYPES > (1 << PB_migratetype_bits)); 526 527 bitmap = get_pageblock_bitmap(page, pfn); 528 bitidx = pfn_to_bitidx(page, pfn); 529 word_bitidx = bitidx / BITS_PER_LONG; 530 bitidx &= (BITS_PER_LONG-1); 531 532 VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page); 533 534 mask <<= bitidx; 535 flags <<= bitidx; 536 537 word = READ_ONCE(bitmap[word_bitidx]); 538 for (;;) { 539 old_word = cmpxchg(&bitmap[word_bitidx], word, (word & ~mask) | flags); 540 if (word == old_word) 541 break; 542 word = old_word; 543 } 544 } 545 546 void set_pageblock_migratetype(struct page *page, int migratetype) 547 { 548 if (unlikely(page_group_by_mobility_disabled && 549 migratetype < MIGRATE_PCPTYPES)) 550 migratetype = MIGRATE_UNMOVABLE; 551 552 set_pfnblock_flags_mask(page, (unsigned long)migratetype, 553 page_to_pfn(page), MIGRATETYPE_MASK); 554 } 555 556 #ifdef CONFIG_DEBUG_VM 557 static int page_outside_zone_boundaries(struct zone *zone, struct page *page) 558 { 559 int ret = 0; 560 unsigned seq; 561 unsigned long pfn = page_to_pfn(page); 562 unsigned long sp, start_pfn; 563 564 do { 565 seq = zone_span_seqbegin(zone); 566 start_pfn = zone->zone_start_pfn; 567 sp = zone->spanned_pages; 568 if (!zone_spans_pfn(zone, pfn)) 569 ret = 1; 570 } while (zone_span_seqretry(zone, seq)); 571 572 if (ret) 573 pr_err("page 0x%lx outside node %d zone %s [ 0x%lx - 0x%lx ]\n", 574 pfn, zone_to_nid(zone), zone->name, 575 start_pfn, start_pfn + sp); 576 577 return ret; 578 } 579 580 static int page_is_consistent(struct zone *zone, struct page *page) 581 { 582 if (zone != page_zone(page)) 583 return 0; 584 585 return 1; 586 } 587 /* 588 * Temporary debugging check for pages not lying within a given zone. 589 */ 590 static int __maybe_unused bad_range(struct zone *zone, struct page *page) 591 { 592 if (page_outside_zone_boundaries(zone, page)) 593 return 1; 594 if (!page_is_consistent(zone, page)) 595 return 1; 596 597 return 0; 598 } 599 #else 600 static inline int __maybe_unused bad_range(struct zone *zone, struct page *page) 601 { 602 return 0; 603 } 604 #endif 605 606 static void bad_page(struct page *page, const char *reason) 607 { 608 static unsigned long resume; 609 static unsigned long nr_shown; 610 static unsigned long nr_unshown; 611 612 /* 613 * Allow a burst of 60 reports, then keep quiet for that minute; 614 * or allow a steady drip of one report per second. 615 */ 616 if (nr_shown == 60) { 617 if (time_before(jiffies, resume)) { 618 nr_unshown++; 619 goto out; 620 } 621 if (nr_unshown) { 622 pr_alert( 623 "BUG: Bad page state: %lu messages suppressed\n", 624 nr_unshown); 625 nr_unshown = 0; 626 } 627 nr_shown = 0; 628 } 629 if (nr_shown++ == 0) 630 resume = jiffies + 60 * HZ; 631 632 pr_alert("BUG: Bad page state in process %s pfn:%05lx\n", 633 current->comm, page_to_pfn(page)); 634 dump_page(page, reason); 635 636 print_modules(); 637 dump_stack(); 638 out: 639 /* Leave bad fields for debug, except PageBuddy could make trouble */ 640 page_mapcount_reset(page); /* remove PageBuddy */ 641 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); 642 } 643 644 static inline unsigned int order_to_pindex(int migratetype, int order) 645 { 646 int base = order; 647 648 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 649 if (order > PAGE_ALLOC_COSTLY_ORDER) { 650 VM_BUG_ON(order != pageblock_order); 651 base = PAGE_ALLOC_COSTLY_ORDER + 1; 652 } 653 #else 654 VM_BUG_ON(order > PAGE_ALLOC_COSTLY_ORDER); 655 #endif 656 657 return (MIGRATE_PCPTYPES * base) + migratetype; 658 } 659 660 static inline int pindex_to_order(unsigned int pindex) 661 { 662 int order = pindex / MIGRATE_PCPTYPES; 663 664 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 665 if (order > PAGE_ALLOC_COSTLY_ORDER) 666 order = pageblock_order; 667 #else 668 VM_BUG_ON(order > PAGE_ALLOC_COSTLY_ORDER); 669 #endif 670 671 return order; 672 } 673 674 static inline bool pcp_allowed_order(unsigned int order) 675 { 676 if (order <= PAGE_ALLOC_COSTLY_ORDER) 677 return true; 678 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 679 if (order == pageblock_order) 680 return true; 681 #endif 682 return false; 683 } 684 685 static inline void free_the_page(struct page *page, unsigned int order) 686 { 687 if (pcp_allowed_order(order)) /* Via pcp? */ 688 free_unref_page(page, order); 689 else 690 __free_pages_ok(page, order, FPI_NONE); 691 } 692 693 /* 694 * Higher-order pages are called "compound pages". They are structured thusly: 695 * 696 * The first PAGE_SIZE page is called the "head page" and have PG_head set. 697 * 698 * The remaining PAGE_SIZE pages are called "tail pages". PageTail() is encoded 699 * in bit 0 of page->compound_head. The rest of bits is pointer to head page. 700 * 701 * The first tail page's ->compound_dtor holds the offset in array of compound 702 * page destructors. See compound_page_dtors. 703 * 704 * The first tail page's ->compound_order holds the order of allocation. 705 * This usage means that zero-order pages may not be compound. 706 */ 707 708 void free_compound_page(struct page *page) 709 { 710 mem_cgroup_uncharge(page_folio(page)); 711 free_the_page(page, compound_order(page)); 712 } 713 714 static void prep_compound_head(struct page *page, unsigned int order) 715 { 716 set_compound_page_dtor(page, COMPOUND_PAGE_DTOR); 717 set_compound_order(page, order); 718 atomic_set(compound_mapcount_ptr(page), -1); 719 atomic_set(compound_pincount_ptr(page), 0); 720 } 721 722 static void prep_compound_tail(struct page *head, int tail_idx) 723 { 724 struct page *p = head + tail_idx; 725 726 p->mapping = TAIL_MAPPING; 727 set_compound_head(p, head); 728 } 729 730 void prep_compound_page(struct page *page, unsigned int order) 731 { 732 int i; 733 int nr_pages = 1 << order; 734 735 __SetPageHead(page); 736 for (i = 1; i < nr_pages; i++) 737 prep_compound_tail(page, i); 738 739 prep_compound_head(page, order); 740 } 741 742 #ifdef CONFIG_DEBUG_PAGEALLOC 743 unsigned int _debug_guardpage_minorder; 744 745 bool _debug_pagealloc_enabled_early __read_mostly 746 = IS_ENABLED(CONFIG_DEBUG_PAGEALLOC_ENABLE_DEFAULT); 747 EXPORT_SYMBOL(_debug_pagealloc_enabled_early); 748 DEFINE_STATIC_KEY_FALSE(_debug_pagealloc_enabled); 749 EXPORT_SYMBOL(_debug_pagealloc_enabled); 750 751 DEFINE_STATIC_KEY_FALSE(_debug_guardpage_enabled); 752 753 static int __init early_debug_pagealloc(char *buf) 754 { 755 return kstrtobool(buf, &_debug_pagealloc_enabled_early); 756 } 757 early_param("debug_pagealloc", early_debug_pagealloc); 758 759 static int __init debug_guardpage_minorder_setup(char *buf) 760 { 761 unsigned long res; 762 763 if (kstrtoul(buf, 10, &res) < 0 || res > MAX_ORDER / 2) { 764 pr_err("Bad debug_guardpage_minorder value\n"); 765 return 0; 766 } 767 _debug_guardpage_minorder = res; 768 pr_info("Setting debug_guardpage_minorder to %lu\n", res); 769 return 0; 770 } 771 early_param("debug_guardpage_minorder", debug_guardpage_minorder_setup); 772 773 static inline bool set_page_guard(struct zone *zone, struct page *page, 774 unsigned int order, int migratetype) 775 { 776 if (!debug_guardpage_enabled()) 777 return false; 778 779 if (order >= debug_guardpage_minorder()) 780 return false; 781 782 __SetPageGuard(page); 783 INIT_LIST_HEAD(&page->lru); 784 set_page_private(page, order); 785 /* Guard pages are not available for any usage */ 786 __mod_zone_freepage_state(zone, -(1 << order), migratetype); 787 788 return true; 789 } 790 791 static inline void clear_page_guard(struct zone *zone, struct page *page, 792 unsigned int order, int migratetype) 793 { 794 if (!debug_guardpage_enabled()) 795 return; 796 797 __ClearPageGuard(page); 798 799 set_page_private(page, 0); 800 if (!is_migrate_isolate(migratetype)) 801 __mod_zone_freepage_state(zone, (1 << order), migratetype); 802 } 803 #else 804 static inline bool set_page_guard(struct zone *zone, struct page *page, 805 unsigned int order, int migratetype) { return false; } 806 static inline void clear_page_guard(struct zone *zone, struct page *page, 807 unsigned int order, int migratetype) {} 808 #endif 809 810 /* 811 * Enable static keys related to various memory debugging and hardening options. 812 * Some override others, and depend on early params that are evaluated in the 813 * order of appearance. So we need to first gather the full picture of what was 814 * enabled, and then make decisions. 815 */ 816 void init_mem_debugging_and_hardening(void) 817 { 818 bool page_poisoning_requested = false; 819 820 #ifdef CONFIG_PAGE_POISONING 821 /* 822 * Page poisoning is debug page alloc for some arches. If 823 * either of those options are enabled, enable poisoning. 824 */ 825 if (page_poisoning_enabled() || 826 (!IS_ENABLED(CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC) && 827 debug_pagealloc_enabled())) { 828 static_branch_enable(&_page_poisoning_enabled); 829 page_poisoning_requested = true; 830 } 831 #endif 832 833 if ((_init_on_alloc_enabled_early || _init_on_free_enabled_early) && 834 page_poisoning_requested) { 835 pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, " 836 "will take precedence over init_on_alloc and init_on_free\n"); 837 _init_on_alloc_enabled_early = false; 838 _init_on_free_enabled_early = false; 839 } 840 841 if (_init_on_alloc_enabled_early) 842 static_branch_enable(&init_on_alloc); 843 else 844 static_branch_disable(&init_on_alloc); 845 846 if (_init_on_free_enabled_early) 847 static_branch_enable(&init_on_free); 848 else 849 static_branch_disable(&init_on_free); 850 851 #ifdef CONFIG_DEBUG_PAGEALLOC 852 if (!debug_pagealloc_enabled()) 853 return; 854 855 static_branch_enable(&_debug_pagealloc_enabled); 856 857 if (!debug_guardpage_minorder()) 858 return; 859 860 static_branch_enable(&_debug_guardpage_enabled); 861 #endif 862 } 863 864 static inline void set_buddy_order(struct page *page, unsigned int order) 865 { 866 set_page_private(page, order); 867 __SetPageBuddy(page); 868 } 869 870 /* 871 * This function checks whether a page is free && is the buddy 872 * we can coalesce a page and its buddy if 873 * (a) the buddy is not in a hole (check before calling!) && 874 * (b) the buddy is in the buddy system && 875 * (c) a page and its buddy have the same order && 876 * (d) a page and its buddy are in the same zone. 877 * 878 * For recording whether a page is in the buddy system, we set PageBuddy. 879 * Setting, clearing, and testing PageBuddy is serialized by zone->lock. 880 * 881 * For recording page's order, we use page_private(page). 882 */ 883 static inline bool page_is_buddy(struct page *page, struct page *buddy, 884 unsigned int order) 885 { 886 if (!page_is_guard(buddy) && !PageBuddy(buddy)) 887 return false; 888 889 if (buddy_order(buddy) != order) 890 return false; 891 892 /* 893 * zone check is done late to avoid uselessly calculating 894 * zone/node ids for pages that could never merge. 895 */ 896 if (page_zone_id(page) != page_zone_id(buddy)) 897 return false; 898 899 VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy); 900 901 return true; 902 } 903 904 #ifdef CONFIG_COMPACTION 905 static inline struct capture_control *task_capc(struct zone *zone) 906 { 907 struct capture_control *capc = current->capture_control; 908 909 return unlikely(capc) && 910 !(current->flags & PF_KTHREAD) && 911 !capc->page && 912 capc->cc->zone == zone ? capc : NULL; 913 } 914 915 static inline bool 916 compaction_capture(struct capture_control *capc, struct page *page, 917 int order, int migratetype) 918 { 919 if (!capc || order != capc->cc->order) 920 return false; 921 922 /* Do not accidentally pollute CMA or isolated regions*/ 923 if (is_migrate_cma(migratetype) || 924 is_migrate_isolate(migratetype)) 925 return false; 926 927 /* 928 * Do not let lower order allocations pollute a movable pageblock. 929 * This might let an unmovable request use a reclaimable pageblock 930 * and vice-versa but no more than normal fallback logic which can 931 * have trouble finding a high-order free page. 932 */ 933 if (order < pageblock_order && migratetype == MIGRATE_MOVABLE) 934 return false; 935 936 capc->page = page; 937 return true; 938 } 939 940 #else 941 static inline struct capture_control *task_capc(struct zone *zone) 942 { 943 return NULL; 944 } 945 946 static inline bool 947 compaction_capture(struct capture_control *capc, struct page *page, 948 int order, int migratetype) 949 { 950 return false; 951 } 952 #endif /* CONFIG_COMPACTION */ 953 954 /* Used for pages not on another list */ 955 static inline void add_to_free_list(struct page *page, struct zone *zone, 956 unsigned int order, int migratetype) 957 { 958 struct free_area *area = &zone->free_area[order]; 959 960 list_add(&page->lru, &area->free_list[migratetype]); 961 area->nr_free++; 962 } 963 964 /* Used for pages not on another list */ 965 static inline void add_to_free_list_tail(struct page *page, struct zone *zone, 966 unsigned int order, int migratetype) 967 { 968 struct free_area *area = &zone->free_area[order]; 969 970 list_add_tail(&page->lru, &area->free_list[migratetype]); 971 area->nr_free++; 972 } 973 974 /* 975 * Used for pages which are on another list. Move the pages to the tail 976 * of the list - so the moved pages won't immediately be considered for 977 * allocation again (e.g., optimization for memory onlining). 978 */ 979 static inline void move_to_free_list(struct page *page, struct zone *zone, 980 unsigned int order, int migratetype) 981 { 982 struct free_area *area = &zone->free_area[order]; 983 984 list_move_tail(&page->lru, &area->free_list[migratetype]); 985 } 986 987 static inline void del_page_from_free_list(struct page *page, struct zone *zone, 988 unsigned int order) 989 { 990 /* clear reported state and update reported page count */ 991 if (page_reported(page)) 992 __ClearPageReported(page); 993 994 list_del(&page->lru); 995 __ClearPageBuddy(page); 996 set_page_private(page, 0); 997 zone->free_area[order].nr_free--; 998 } 999 1000 /* 1001 * If this is not the largest possible page, check if the buddy 1002 * of the next-highest order is free. If it is, it's possible 1003 * that pages are being freed that will coalesce soon. In case, 1004 * that is happening, add the free page to the tail of the list 1005 * so it's less likely to be used soon and more likely to be merged 1006 * as a higher order page 1007 */ 1008 static inline bool 1009 buddy_merge_likely(unsigned long pfn, unsigned long buddy_pfn, 1010 struct page *page, unsigned int order) 1011 { 1012 struct page *higher_page, *higher_buddy; 1013 unsigned long combined_pfn; 1014 1015 if (order >= MAX_ORDER - 2) 1016 return false; 1017 1018 combined_pfn = buddy_pfn & pfn; 1019 higher_page = page + (combined_pfn - pfn); 1020 buddy_pfn = __find_buddy_pfn(combined_pfn, order + 1); 1021 higher_buddy = higher_page + (buddy_pfn - combined_pfn); 1022 1023 return page_is_buddy(higher_page, higher_buddy, order + 1); 1024 } 1025 1026 /* 1027 * Freeing function for a buddy system allocator. 1028 * 1029 * The concept of a buddy system is to maintain direct-mapped table 1030 * (containing bit values) for memory blocks of various "orders". 1031 * The bottom level table contains the map for the smallest allocatable 1032 * units of memory (here, pages), and each level above it describes 1033 * pairs of units from the levels below, hence, "buddies". 1034 * At a high level, all that happens here is marking the table entry 1035 * at the bottom level available, and propagating the changes upward 1036 * as necessary, plus some accounting needed to play nicely with other 1037 * parts of the VM system. 1038 * At each level, we keep a list of pages, which are heads of continuous 1039 * free pages of length of (1 << order) and marked with PageBuddy. 1040 * Page's order is recorded in page_private(page) field. 1041 * So when we are allocating or freeing one, we can derive the state of the 1042 * other. That is, if we allocate a small block, and both were 1043 * free, the remainder of the region must be split into blocks. 1044 * If a block is freed, and its buddy is also free, then this 1045 * triggers coalescing into a block of larger size. 1046 * 1047 * -- nyc 1048 */ 1049 1050 static inline void __free_one_page(struct page *page, 1051 unsigned long pfn, 1052 struct zone *zone, unsigned int order, 1053 int migratetype, fpi_t fpi_flags) 1054 { 1055 struct capture_control *capc = task_capc(zone); 1056 unsigned int max_order = pageblock_order; 1057 unsigned long buddy_pfn; 1058 unsigned long combined_pfn; 1059 struct page *buddy; 1060 bool to_tail; 1061 1062 VM_BUG_ON(!zone_is_initialized(zone)); 1063 VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page); 1064 1065 VM_BUG_ON(migratetype == -1); 1066 if (likely(!is_migrate_isolate(migratetype))) 1067 __mod_zone_freepage_state(zone, 1 << order, migratetype); 1068 1069 VM_BUG_ON_PAGE(pfn & ((1 << order) - 1), page); 1070 VM_BUG_ON_PAGE(bad_range(zone, page), page); 1071 1072 continue_merging: 1073 while (order < max_order) { 1074 if (compaction_capture(capc, page, order, migratetype)) { 1075 __mod_zone_freepage_state(zone, -(1 << order), 1076 migratetype); 1077 return; 1078 } 1079 buddy_pfn = __find_buddy_pfn(pfn, order); 1080 buddy = page + (buddy_pfn - pfn); 1081 1082 if (!page_is_buddy(page, buddy, order)) 1083 goto done_merging; 1084 /* 1085 * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page, 1086 * merge with it and move up one order. 1087 */ 1088 if (page_is_guard(buddy)) 1089 clear_page_guard(zone, buddy, order, migratetype); 1090 else 1091 del_page_from_free_list(buddy, zone, order); 1092 combined_pfn = buddy_pfn & pfn; 1093 page = page + (combined_pfn - pfn); 1094 pfn = combined_pfn; 1095 order++; 1096 } 1097 if (order < MAX_ORDER - 1) { 1098 /* If we are here, it means order is >= pageblock_order. 1099 * We want to prevent merge between freepages on pageblock 1100 * without fallbacks and normal pageblock. Without this, 1101 * pageblock isolation could cause incorrect freepage or CMA 1102 * accounting or HIGHATOMIC accounting. 1103 * 1104 * We don't want to hit this code for the more frequent 1105 * low-order merging. 1106 */ 1107 int buddy_mt; 1108 1109 buddy_pfn = __find_buddy_pfn(pfn, order); 1110 buddy = page + (buddy_pfn - pfn); 1111 1112 if (!page_is_buddy(page, buddy, order)) 1113 goto done_merging; 1114 buddy_mt = get_pageblock_migratetype(buddy); 1115 1116 if (migratetype != buddy_mt 1117 && (!migratetype_is_mergeable(migratetype) || 1118 !migratetype_is_mergeable(buddy_mt))) 1119 goto done_merging; 1120 max_order = order + 1; 1121 goto continue_merging; 1122 } 1123 1124 done_merging: 1125 set_buddy_order(page, order); 1126 1127 if (fpi_flags & FPI_TO_TAIL) 1128 to_tail = true; 1129 else if (is_shuffle_order(order)) 1130 to_tail = shuffle_pick_tail(); 1131 else 1132 to_tail = buddy_merge_likely(pfn, buddy_pfn, page, order); 1133 1134 if (to_tail) 1135 add_to_free_list_tail(page, zone, order, migratetype); 1136 else 1137 add_to_free_list(page, zone, order, migratetype); 1138 1139 /* Notify page reporting subsystem of freed page */ 1140 if (!(fpi_flags & FPI_SKIP_REPORT_NOTIFY)) 1141 page_reporting_notify_free(order); 1142 } 1143 1144 /* 1145 * A bad page could be due to a number of fields. Instead of multiple branches, 1146 * try and check multiple fields with one check. The caller must do a detailed 1147 * check if necessary. 1148 */ 1149 static inline bool page_expected_state(struct page *page, 1150 unsigned long check_flags) 1151 { 1152 if (unlikely(atomic_read(&page->_mapcount) != -1)) 1153 return false; 1154 1155 if (unlikely((unsigned long)page->mapping | 1156 page_ref_count(page) | 1157 #ifdef CONFIG_MEMCG 1158 page->memcg_data | 1159 #endif 1160 (page->flags & check_flags))) 1161 return false; 1162 1163 return true; 1164 } 1165 1166 static const char *page_bad_reason(struct page *page, unsigned long flags) 1167 { 1168 const char *bad_reason = NULL; 1169 1170 if (unlikely(atomic_read(&page->_mapcount) != -1)) 1171 bad_reason = "nonzero mapcount"; 1172 if (unlikely(page->mapping != NULL)) 1173 bad_reason = "non-NULL mapping"; 1174 if (unlikely(page_ref_count(page) != 0)) 1175 bad_reason = "nonzero _refcount"; 1176 if (unlikely(page->flags & flags)) { 1177 if (flags == PAGE_FLAGS_CHECK_AT_PREP) 1178 bad_reason = "PAGE_FLAGS_CHECK_AT_PREP flag(s) set"; 1179 else 1180 bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set"; 1181 } 1182 #ifdef CONFIG_MEMCG 1183 if (unlikely(page->memcg_data)) 1184 bad_reason = "page still charged to cgroup"; 1185 #endif 1186 return bad_reason; 1187 } 1188 1189 static void check_free_page_bad(struct page *page) 1190 { 1191 bad_page(page, 1192 page_bad_reason(page, PAGE_FLAGS_CHECK_AT_FREE)); 1193 } 1194 1195 static inline int check_free_page(struct page *page) 1196 { 1197 if (likely(page_expected_state(page, PAGE_FLAGS_CHECK_AT_FREE))) 1198 return 0; 1199 1200 /* Something has gone sideways, find it */ 1201 check_free_page_bad(page); 1202 return 1; 1203 } 1204 1205 static int free_tail_pages_check(struct page *head_page, struct page *page) 1206 { 1207 int ret = 1; 1208 1209 /* 1210 * We rely page->lru.next never has bit 0 set, unless the page 1211 * is PageTail(). Let's make sure that's true even for poisoned ->lru. 1212 */ 1213 BUILD_BUG_ON((unsigned long)LIST_POISON1 & 1); 1214 1215 if (!IS_ENABLED(CONFIG_DEBUG_VM)) { 1216 ret = 0; 1217 goto out; 1218 } 1219 switch (page - head_page) { 1220 case 1: 1221 /* the first tail page: ->mapping may be compound_mapcount() */ 1222 if (unlikely(compound_mapcount(page))) { 1223 bad_page(page, "nonzero compound_mapcount"); 1224 goto out; 1225 } 1226 break; 1227 case 2: 1228 /* 1229 * the second tail page: ->mapping is 1230 * deferred_list.next -- ignore value. 1231 */ 1232 break; 1233 default: 1234 if (page->mapping != TAIL_MAPPING) { 1235 bad_page(page, "corrupted mapping in tail page"); 1236 goto out; 1237 } 1238 break; 1239 } 1240 if (unlikely(!PageTail(page))) { 1241 bad_page(page, "PageTail not set"); 1242 goto out; 1243 } 1244 if (unlikely(compound_head(page) != head_page)) { 1245 bad_page(page, "compound_head not consistent"); 1246 goto out; 1247 } 1248 ret = 0; 1249 out: 1250 page->mapping = NULL; 1251 clear_compound_head(page); 1252 return ret; 1253 } 1254 1255 /* 1256 * Skip KASAN memory poisoning when either: 1257 * 1258 * 1. Deferred memory initialization has not yet completed, 1259 * see the explanation below. 1260 * 2. Skipping poisoning is requested via FPI_SKIP_KASAN_POISON, 1261 * see the comment next to it. 1262 * 3. Skipping poisoning is requested via __GFP_SKIP_KASAN_POISON, 1263 * see the comment next to it. 1264 * 1265 * Poisoning pages during deferred memory init will greatly lengthen the 1266 * process and cause problem in large memory systems as the deferred pages 1267 * initialization is done with interrupt disabled. 1268 * 1269 * Assuming that there will be no reference to those newly initialized 1270 * pages before they are ever allocated, this should have no effect on 1271 * KASAN memory tracking as the poison will be properly inserted at page 1272 * allocation time. The only corner case is when pages are allocated by 1273 * on-demand allocation and then freed again before the deferred pages 1274 * initialization is done, but this is not likely to happen. 1275 */ 1276 static inline bool should_skip_kasan_poison(struct page *page, fpi_t fpi_flags) 1277 { 1278 return deferred_pages_enabled() || 1279 (!IS_ENABLED(CONFIG_KASAN_GENERIC) && 1280 (fpi_flags & FPI_SKIP_KASAN_POISON)) || 1281 PageSkipKASanPoison(page); 1282 } 1283 1284 static void kernel_init_free_pages(struct page *page, int numpages) 1285 { 1286 int i; 1287 1288 /* s390's use of memset() could override KASAN redzones. */ 1289 kasan_disable_current(); 1290 for (i = 0; i < numpages; i++) { 1291 u8 tag = page_kasan_tag(page + i); 1292 page_kasan_tag_reset(page + i); 1293 clear_highpage(page + i); 1294 page_kasan_tag_set(page + i, tag); 1295 } 1296 kasan_enable_current(); 1297 } 1298 1299 static __always_inline bool free_pages_prepare(struct page *page, 1300 unsigned int order, bool check_free, fpi_t fpi_flags) 1301 { 1302 int bad = 0; 1303 bool init = want_init_on_free(); 1304 1305 VM_BUG_ON_PAGE(PageTail(page), page); 1306 1307 trace_mm_page_free(page, order); 1308 1309 if (unlikely(PageHWPoison(page)) && !order) { 1310 /* 1311 * Do not let hwpoison pages hit pcplists/buddy 1312 * Untie memcg state and reset page's owner 1313 */ 1314 if (memcg_kmem_enabled() && PageMemcgKmem(page)) 1315 __memcg_kmem_uncharge_page(page, order); 1316 reset_page_owner(page, order); 1317 page_table_check_free(page, order); 1318 return false; 1319 } 1320 1321 /* 1322 * Check tail pages before head page information is cleared to 1323 * avoid checking PageCompound for order-0 pages. 1324 */ 1325 if (unlikely(order)) { 1326 bool compound = PageCompound(page); 1327 int i; 1328 1329 VM_BUG_ON_PAGE(compound && compound_order(page) != order, page); 1330 1331 if (compound) { 1332 ClearPageDoubleMap(page); 1333 ClearPageHasHWPoisoned(page); 1334 } 1335 for (i = 1; i < (1 << order); i++) { 1336 if (compound) 1337 bad += free_tail_pages_check(page, page + i); 1338 if (unlikely(check_free_page(page + i))) { 1339 bad++; 1340 continue; 1341 } 1342 (page + i)->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; 1343 } 1344 } 1345 if (PageMappingFlags(page)) 1346 page->mapping = NULL; 1347 if (memcg_kmem_enabled() && PageMemcgKmem(page)) 1348 __memcg_kmem_uncharge_page(page, order); 1349 if (check_free) 1350 bad += check_free_page(page); 1351 if (bad) 1352 return false; 1353 1354 page_cpupid_reset_last(page); 1355 page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; 1356 reset_page_owner(page, order); 1357 page_table_check_free(page, order); 1358 1359 if (!PageHighMem(page)) { 1360 debug_check_no_locks_freed(page_address(page), 1361 PAGE_SIZE << order); 1362 debug_check_no_obj_freed(page_address(page), 1363 PAGE_SIZE << order); 1364 } 1365 1366 kernel_poison_pages(page, 1 << order); 1367 1368 /* 1369 * As memory initialization might be integrated into KASAN, 1370 * KASAN poisoning and memory initialization code must be 1371 * kept together to avoid discrepancies in behavior. 1372 * 1373 * With hardware tag-based KASAN, memory tags must be set before the 1374 * page becomes unavailable via debug_pagealloc or arch_free_page. 1375 */ 1376 if (!should_skip_kasan_poison(page, fpi_flags)) { 1377 kasan_poison_pages(page, order, init); 1378 1379 /* Memory is already initialized if KASAN did it internally. */ 1380 if (kasan_has_integrated_init()) 1381 init = false; 1382 } 1383 if (init) 1384 kernel_init_free_pages(page, 1 << order); 1385 1386 /* 1387 * arch_free_page() can make the page's contents inaccessible. s390 1388 * does this. So nothing which can access the page's contents should 1389 * happen after this. 1390 */ 1391 arch_free_page(page, order); 1392 1393 debug_pagealloc_unmap_pages(page, 1 << order); 1394 1395 return true; 1396 } 1397 1398 #ifdef CONFIG_DEBUG_VM 1399 /* 1400 * With DEBUG_VM enabled, order-0 pages are checked immediately when being freed 1401 * to pcp lists. With debug_pagealloc also enabled, they are also rechecked when 1402 * moved from pcp lists to free lists. 1403 */ 1404 static bool free_pcp_prepare(struct page *page, unsigned int order) 1405 { 1406 return free_pages_prepare(page, order, true, FPI_NONE); 1407 } 1408 1409 static bool bulkfree_pcp_prepare(struct page *page) 1410 { 1411 if (debug_pagealloc_enabled_static()) 1412 return check_free_page(page); 1413 else 1414 return false; 1415 } 1416 #else 1417 /* 1418 * With DEBUG_VM disabled, order-0 pages being freed are checked only when 1419 * moving from pcp lists to free list in order to reduce overhead. With 1420 * debug_pagealloc enabled, they are checked also immediately when being freed 1421 * to the pcp lists. 1422 */ 1423 static bool free_pcp_prepare(struct page *page, unsigned int order) 1424 { 1425 if (debug_pagealloc_enabled_static()) 1426 return free_pages_prepare(page, order, true, FPI_NONE); 1427 else 1428 return free_pages_prepare(page, order, false, FPI_NONE); 1429 } 1430 1431 static bool bulkfree_pcp_prepare(struct page *page) 1432 { 1433 return check_free_page(page); 1434 } 1435 #endif /* CONFIG_DEBUG_VM */ 1436 1437 /* 1438 * Frees a number of pages from the PCP lists 1439 * Assumes all pages on list are in same zone. 1440 * count is the number of pages to free. 1441 */ 1442 static void free_pcppages_bulk(struct zone *zone, int count, 1443 struct per_cpu_pages *pcp, 1444 int pindex) 1445 { 1446 int min_pindex = 0; 1447 int max_pindex = NR_PCP_LISTS - 1; 1448 unsigned int order; 1449 bool isolated_pageblocks; 1450 struct page *page; 1451 1452 /* 1453 * Ensure proper count is passed which otherwise would stuck in the 1454 * below while (list_empty(list)) loop. 1455 */ 1456 count = min(pcp->count, count); 1457 1458 /* Ensure requested pindex is drained first. */ 1459 pindex = pindex - 1; 1460 1461 /* 1462 * local_lock_irq held so equivalent to spin_lock_irqsave for 1463 * both PREEMPT_RT and non-PREEMPT_RT configurations. 1464 */ 1465 spin_lock(&zone->lock); 1466 isolated_pageblocks = has_isolate_pageblock(zone); 1467 1468 while (count > 0) { 1469 struct list_head *list; 1470 int nr_pages; 1471 1472 /* Remove pages from lists in a round-robin fashion. */ 1473 do { 1474 if (++pindex > max_pindex) 1475 pindex = min_pindex; 1476 list = &pcp->lists[pindex]; 1477 if (!list_empty(list)) 1478 break; 1479 1480 if (pindex == max_pindex) 1481 max_pindex--; 1482 if (pindex == min_pindex) 1483 min_pindex++; 1484 } while (1); 1485 1486 order = pindex_to_order(pindex); 1487 nr_pages = 1 << order; 1488 BUILD_BUG_ON(MAX_ORDER >= (1<<NR_PCP_ORDER_WIDTH)); 1489 do { 1490 int mt; 1491 1492 page = list_last_entry(list, struct page, lru); 1493 mt = get_pcppage_migratetype(page); 1494 1495 /* must delete to avoid corrupting pcp list */ 1496 list_del(&page->lru); 1497 count -= nr_pages; 1498 pcp->count -= nr_pages; 1499 1500 if (bulkfree_pcp_prepare(page)) 1501 continue; 1502 1503 /* MIGRATE_ISOLATE page should not go to pcplists */ 1504 VM_BUG_ON_PAGE(is_migrate_isolate(mt), page); 1505 /* Pageblock could have been isolated meanwhile */ 1506 if (unlikely(isolated_pageblocks)) 1507 mt = get_pageblock_migratetype(page); 1508 1509 __free_one_page(page, page_to_pfn(page), zone, order, mt, FPI_NONE); 1510 trace_mm_page_pcpu_drain(page, order, mt); 1511 } while (count > 0 && !list_empty(list)); 1512 } 1513 1514 spin_unlock(&zone->lock); 1515 } 1516 1517 static void free_one_page(struct zone *zone, 1518 struct page *page, unsigned long pfn, 1519 unsigned int order, 1520 int migratetype, fpi_t fpi_flags) 1521 { 1522 unsigned long flags; 1523 1524 spin_lock_irqsave(&zone->lock, flags); 1525 if (unlikely(has_isolate_pageblock(zone) || 1526 is_migrate_isolate(migratetype))) { 1527 migratetype = get_pfnblock_migratetype(page, pfn); 1528 } 1529 __free_one_page(page, pfn, zone, order, migratetype, fpi_flags); 1530 spin_unlock_irqrestore(&zone->lock, flags); 1531 } 1532 1533 static void __meminit __init_single_page(struct page *page, unsigned long pfn, 1534 unsigned long zone, int nid) 1535 { 1536 mm_zero_struct_page(page); 1537 set_page_links(page, zone, nid, pfn); 1538 init_page_count(page); 1539 page_mapcount_reset(page); 1540 page_cpupid_reset_last(page); 1541 page_kasan_tag_reset(page); 1542 1543 INIT_LIST_HEAD(&page->lru); 1544 #ifdef WANT_PAGE_VIRTUAL 1545 /* The shift won't overflow because ZONE_NORMAL is below 4G. */ 1546 if (!is_highmem_idx(zone)) 1547 set_page_address(page, __va(pfn << PAGE_SHIFT)); 1548 #endif 1549 } 1550 1551 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1552 static void __meminit init_reserved_page(unsigned long pfn) 1553 { 1554 pg_data_t *pgdat; 1555 int nid, zid; 1556 1557 if (!early_page_uninitialised(pfn)) 1558 return; 1559 1560 nid = early_pfn_to_nid(pfn); 1561 pgdat = NODE_DATA(nid); 1562 1563 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 1564 struct zone *zone = &pgdat->node_zones[zid]; 1565 1566 if (zone_spans_pfn(zone, pfn)) 1567 break; 1568 } 1569 __init_single_page(pfn_to_page(pfn), pfn, zid, nid); 1570 } 1571 #else 1572 static inline void init_reserved_page(unsigned long pfn) 1573 { 1574 } 1575 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ 1576 1577 /* 1578 * Initialised pages do not have PageReserved set. This function is 1579 * called for each range allocated by the bootmem allocator and 1580 * marks the pages PageReserved. The remaining valid pages are later 1581 * sent to the buddy page allocator. 1582 */ 1583 void __meminit reserve_bootmem_region(phys_addr_t start, phys_addr_t end) 1584 { 1585 unsigned long start_pfn = PFN_DOWN(start); 1586 unsigned long end_pfn = PFN_UP(end); 1587 1588 for (; start_pfn < end_pfn; start_pfn++) { 1589 if (pfn_valid(start_pfn)) { 1590 struct page *page = pfn_to_page(start_pfn); 1591 1592 init_reserved_page(start_pfn); 1593 1594 /* Avoid false-positive PageTail() */ 1595 INIT_LIST_HEAD(&page->lru); 1596 1597 /* 1598 * no need for atomic set_bit because the struct 1599 * page is not visible yet so nobody should 1600 * access it yet. 1601 */ 1602 __SetPageReserved(page); 1603 } 1604 } 1605 } 1606 1607 static void __free_pages_ok(struct page *page, unsigned int order, 1608 fpi_t fpi_flags) 1609 { 1610 unsigned long flags; 1611 int migratetype; 1612 unsigned long pfn = page_to_pfn(page); 1613 struct zone *zone = page_zone(page); 1614 1615 if (!free_pages_prepare(page, order, true, fpi_flags)) 1616 return; 1617 1618 migratetype = get_pfnblock_migratetype(page, pfn); 1619 1620 spin_lock_irqsave(&zone->lock, flags); 1621 if (unlikely(has_isolate_pageblock(zone) || 1622 is_migrate_isolate(migratetype))) { 1623 migratetype = get_pfnblock_migratetype(page, pfn); 1624 } 1625 __free_one_page(page, pfn, zone, order, migratetype, fpi_flags); 1626 spin_unlock_irqrestore(&zone->lock, flags); 1627 1628 __count_vm_events(PGFREE, 1 << order); 1629 } 1630 1631 void __free_pages_core(struct page *page, unsigned int order) 1632 { 1633 unsigned int nr_pages = 1 << order; 1634 struct page *p = page; 1635 unsigned int loop; 1636 1637 /* 1638 * When initializing the memmap, __init_single_page() sets the refcount 1639 * of all pages to 1 ("allocated"/"not free"). We have to set the 1640 * refcount of all involved pages to 0. 1641 */ 1642 prefetchw(p); 1643 for (loop = 0; loop < (nr_pages - 1); loop++, p++) { 1644 prefetchw(p + 1); 1645 __ClearPageReserved(p); 1646 set_page_count(p, 0); 1647 } 1648 __ClearPageReserved(p); 1649 set_page_count(p, 0); 1650 1651 atomic_long_add(nr_pages, &page_zone(page)->managed_pages); 1652 1653 /* 1654 * Bypass PCP and place fresh pages right to the tail, primarily 1655 * relevant for memory onlining. 1656 */ 1657 __free_pages_ok(page, order, FPI_TO_TAIL | FPI_SKIP_KASAN_POISON); 1658 } 1659 1660 #ifdef CONFIG_NUMA 1661 1662 /* 1663 * During memory init memblocks map pfns to nids. The search is expensive and 1664 * this caches recent lookups. The implementation of __early_pfn_to_nid 1665 * treats start/end as pfns. 1666 */ 1667 struct mminit_pfnnid_cache { 1668 unsigned long last_start; 1669 unsigned long last_end; 1670 int last_nid; 1671 }; 1672 1673 static struct mminit_pfnnid_cache early_pfnnid_cache __meminitdata; 1674 1675 /* 1676 * Required by SPARSEMEM. Given a PFN, return what node the PFN is on. 1677 */ 1678 static int __meminit __early_pfn_to_nid(unsigned long pfn, 1679 struct mminit_pfnnid_cache *state) 1680 { 1681 unsigned long start_pfn, end_pfn; 1682 int nid; 1683 1684 if (state->last_start <= pfn && pfn < state->last_end) 1685 return state->last_nid; 1686 1687 nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn); 1688 if (nid != NUMA_NO_NODE) { 1689 state->last_start = start_pfn; 1690 state->last_end = end_pfn; 1691 state->last_nid = nid; 1692 } 1693 1694 return nid; 1695 } 1696 1697 int __meminit early_pfn_to_nid(unsigned long pfn) 1698 { 1699 static DEFINE_SPINLOCK(early_pfn_lock); 1700 int nid; 1701 1702 spin_lock(&early_pfn_lock); 1703 nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache); 1704 if (nid < 0) 1705 nid = first_online_node; 1706 spin_unlock(&early_pfn_lock); 1707 1708 return nid; 1709 } 1710 #endif /* CONFIG_NUMA */ 1711 1712 void __init memblock_free_pages(struct page *page, unsigned long pfn, 1713 unsigned int order) 1714 { 1715 if (early_page_uninitialised(pfn)) 1716 return; 1717 __free_pages_core(page, order); 1718 } 1719 1720 /* 1721 * Check that the whole (or subset of) a pageblock given by the interval of 1722 * [start_pfn, end_pfn) is valid and within the same zone, before scanning it 1723 * with the migration of free compaction scanner. 1724 * 1725 * Return struct page pointer of start_pfn, or NULL if checks were not passed. 1726 * 1727 * It's possible on some configurations to have a setup like node0 node1 node0 1728 * i.e. it's possible that all pages within a zones range of pages do not 1729 * belong to a single zone. We assume that a border between node0 and node1 1730 * can occur within a single pageblock, but not a node0 node1 node0 1731 * interleaving within a single pageblock. It is therefore sufficient to check 1732 * the first and last page of a pageblock and avoid checking each individual 1733 * page in a pageblock. 1734 */ 1735 struct page *__pageblock_pfn_to_page(unsigned long start_pfn, 1736 unsigned long end_pfn, struct zone *zone) 1737 { 1738 struct page *start_page; 1739 struct page *end_page; 1740 1741 /* end_pfn is one past the range we are checking */ 1742 end_pfn--; 1743 1744 if (!pfn_valid(start_pfn) || !pfn_valid(end_pfn)) 1745 return NULL; 1746 1747 start_page = pfn_to_online_page(start_pfn); 1748 if (!start_page) 1749 return NULL; 1750 1751 if (page_zone(start_page) != zone) 1752 return NULL; 1753 1754 end_page = pfn_to_page(end_pfn); 1755 1756 /* This gives a shorter code than deriving page_zone(end_page) */ 1757 if (page_zone_id(start_page) != page_zone_id(end_page)) 1758 return NULL; 1759 1760 return start_page; 1761 } 1762 1763 void set_zone_contiguous(struct zone *zone) 1764 { 1765 unsigned long block_start_pfn = zone->zone_start_pfn; 1766 unsigned long block_end_pfn; 1767 1768 block_end_pfn = ALIGN(block_start_pfn + 1, pageblock_nr_pages); 1769 for (; block_start_pfn < zone_end_pfn(zone); 1770 block_start_pfn = block_end_pfn, 1771 block_end_pfn += pageblock_nr_pages) { 1772 1773 block_end_pfn = min(block_end_pfn, zone_end_pfn(zone)); 1774 1775 if (!__pageblock_pfn_to_page(block_start_pfn, 1776 block_end_pfn, zone)) 1777 return; 1778 cond_resched(); 1779 } 1780 1781 /* We confirm that there is no hole */ 1782 zone->contiguous = true; 1783 } 1784 1785 void clear_zone_contiguous(struct zone *zone) 1786 { 1787 zone->contiguous = false; 1788 } 1789 1790 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1791 static void __init deferred_free_range(unsigned long pfn, 1792 unsigned long nr_pages) 1793 { 1794 struct page *page; 1795 unsigned long i; 1796 1797 if (!nr_pages) 1798 return; 1799 1800 page = pfn_to_page(pfn); 1801 1802 /* Free a large naturally-aligned chunk if possible */ 1803 if (nr_pages == pageblock_nr_pages && 1804 (pfn & (pageblock_nr_pages - 1)) == 0) { 1805 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 1806 __free_pages_core(page, pageblock_order); 1807 return; 1808 } 1809 1810 for (i = 0; i < nr_pages; i++, page++, pfn++) { 1811 if ((pfn & (pageblock_nr_pages - 1)) == 0) 1812 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 1813 __free_pages_core(page, 0); 1814 } 1815 } 1816 1817 /* Completion tracking for deferred_init_memmap() threads */ 1818 static atomic_t pgdat_init_n_undone __initdata; 1819 static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp); 1820 1821 static inline void __init pgdat_init_report_one_done(void) 1822 { 1823 if (atomic_dec_and_test(&pgdat_init_n_undone)) 1824 complete(&pgdat_init_all_done_comp); 1825 } 1826 1827 /* 1828 * Returns true if page needs to be initialized or freed to buddy allocator. 1829 * 1830 * First we check if pfn is valid on architectures where it is possible to have 1831 * holes within pageblock_nr_pages. On systems where it is not possible, this 1832 * function is optimized out. 1833 * 1834 * Then, we check if a current large page is valid by only checking the validity 1835 * of the head pfn. 1836 */ 1837 static inline bool __init deferred_pfn_valid(unsigned long pfn) 1838 { 1839 if (!(pfn & (pageblock_nr_pages - 1)) && !pfn_valid(pfn)) 1840 return false; 1841 return true; 1842 } 1843 1844 /* 1845 * Free pages to buddy allocator. Try to free aligned pages in 1846 * pageblock_nr_pages sizes. 1847 */ 1848 static void __init deferred_free_pages(unsigned long pfn, 1849 unsigned long end_pfn) 1850 { 1851 unsigned long nr_pgmask = pageblock_nr_pages - 1; 1852 unsigned long nr_free = 0; 1853 1854 for (; pfn < end_pfn; pfn++) { 1855 if (!deferred_pfn_valid(pfn)) { 1856 deferred_free_range(pfn - nr_free, nr_free); 1857 nr_free = 0; 1858 } else if (!(pfn & nr_pgmask)) { 1859 deferred_free_range(pfn - nr_free, nr_free); 1860 nr_free = 1; 1861 } else { 1862 nr_free++; 1863 } 1864 } 1865 /* Free the last block of pages to allocator */ 1866 deferred_free_range(pfn - nr_free, nr_free); 1867 } 1868 1869 /* 1870 * Initialize struct pages. We minimize pfn page lookups and scheduler checks 1871 * by performing it only once every pageblock_nr_pages. 1872 * Return number of pages initialized. 1873 */ 1874 static unsigned long __init deferred_init_pages(struct zone *zone, 1875 unsigned long pfn, 1876 unsigned long end_pfn) 1877 { 1878 unsigned long nr_pgmask = pageblock_nr_pages - 1; 1879 int nid = zone_to_nid(zone); 1880 unsigned long nr_pages = 0; 1881 int zid = zone_idx(zone); 1882 struct page *page = NULL; 1883 1884 for (; pfn < end_pfn; pfn++) { 1885 if (!deferred_pfn_valid(pfn)) { 1886 page = NULL; 1887 continue; 1888 } else if (!page || !(pfn & nr_pgmask)) { 1889 page = pfn_to_page(pfn); 1890 } else { 1891 page++; 1892 } 1893 __init_single_page(page, pfn, zid, nid); 1894 nr_pages++; 1895 } 1896 return (nr_pages); 1897 } 1898 1899 /* 1900 * This function is meant to pre-load the iterator for the zone init. 1901 * Specifically it walks through the ranges until we are caught up to the 1902 * first_init_pfn value and exits there. If we never encounter the value we 1903 * return false indicating there are no valid ranges left. 1904 */ 1905 static bool __init 1906 deferred_init_mem_pfn_range_in_zone(u64 *i, struct zone *zone, 1907 unsigned long *spfn, unsigned long *epfn, 1908 unsigned long first_init_pfn) 1909 { 1910 u64 j; 1911 1912 /* 1913 * Start out by walking through the ranges in this zone that have 1914 * already been initialized. We don't need to do anything with them 1915 * so we just need to flush them out of the system. 1916 */ 1917 for_each_free_mem_pfn_range_in_zone(j, zone, spfn, epfn) { 1918 if (*epfn <= first_init_pfn) 1919 continue; 1920 if (*spfn < first_init_pfn) 1921 *spfn = first_init_pfn; 1922 *i = j; 1923 return true; 1924 } 1925 1926 return false; 1927 } 1928 1929 /* 1930 * Initialize and free pages. We do it in two loops: first we initialize 1931 * struct page, then free to buddy allocator, because while we are 1932 * freeing pages we can access pages that are ahead (computing buddy 1933 * page in __free_one_page()). 1934 * 1935 * In order to try and keep some memory in the cache we have the loop 1936 * broken along max page order boundaries. This way we will not cause 1937 * any issues with the buddy page computation. 1938 */ 1939 static unsigned long __init 1940 deferred_init_maxorder(u64 *i, struct zone *zone, unsigned long *start_pfn, 1941 unsigned long *end_pfn) 1942 { 1943 unsigned long mo_pfn = ALIGN(*start_pfn + 1, MAX_ORDER_NR_PAGES); 1944 unsigned long spfn = *start_pfn, epfn = *end_pfn; 1945 unsigned long nr_pages = 0; 1946 u64 j = *i; 1947 1948 /* First we loop through and initialize the page values */ 1949 for_each_free_mem_pfn_range_in_zone_from(j, zone, start_pfn, end_pfn) { 1950 unsigned long t; 1951 1952 if (mo_pfn <= *start_pfn) 1953 break; 1954 1955 t = min(mo_pfn, *end_pfn); 1956 nr_pages += deferred_init_pages(zone, *start_pfn, t); 1957 1958 if (mo_pfn < *end_pfn) { 1959 *start_pfn = mo_pfn; 1960 break; 1961 } 1962 } 1963 1964 /* Reset values and now loop through freeing pages as needed */ 1965 swap(j, *i); 1966 1967 for_each_free_mem_pfn_range_in_zone_from(j, zone, &spfn, &epfn) { 1968 unsigned long t; 1969 1970 if (mo_pfn <= spfn) 1971 break; 1972 1973 t = min(mo_pfn, epfn); 1974 deferred_free_pages(spfn, t); 1975 1976 if (mo_pfn <= epfn) 1977 break; 1978 } 1979 1980 return nr_pages; 1981 } 1982 1983 static void __init 1984 deferred_init_memmap_chunk(unsigned long start_pfn, unsigned long end_pfn, 1985 void *arg) 1986 { 1987 unsigned long spfn, epfn; 1988 struct zone *zone = arg; 1989 u64 i; 1990 1991 deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, start_pfn); 1992 1993 /* 1994 * Initialize and free pages in MAX_ORDER sized increments so that we 1995 * can avoid introducing any issues with the buddy allocator. 1996 */ 1997 while (spfn < end_pfn) { 1998 deferred_init_maxorder(&i, zone, &spfn, &epfn); 1999 cond_resched(); 2000 } 2001 } 2002 2003 /* An arch may override for more concurrency. */ 2004 __weak int __init 2005 deferred_page_init_max_threads(const struct cpumask *node_cpumask) 2006 { 2007 return 1; 2008 } 2009 2010 /* Initialise remaining memory on a node */ 2011 static int __init deferred_init_memmap(void *data) 2012 { 2013 pg_data_t *pgdat = data; 2014 const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id); 2015 unsigned long spfn = 0, epfn = 0; 2016 unsigned long first_init_pfn, flags; 2017 unsigned long start = jiffies; 2018 struct zone *zone; 2019 int zid, max_threads; 2020 u64 i; 2021 2022 /* Bind memory initialisation thread to a local node if possible */ 2023 if (!cpumask_empty(cpumask)) 2024 set_cpus_allowed_ptr(current, cpumask); 2025 2026 pgdat_resize_lock(pgdat, &flags); 2027 first_init_pfn = pgdat->first_deferred_pfn; 2028 if (first_init_pfn == ULONG_MAX) { 2029 pgdat_resize_unlock(pgdat, &flags); 2030 pgdat_init_report_one_done(); 2031 return 0; 2032 } 2033 2034 /* Sanity check boundaries */ 2035 BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn); 2036 BUG_ON(pgdat->first_deferred_pfn > pgdat_end_pfn(pgdat)); 2037 pgdat->first_deferred_pfn = ULONG_MAX; 2038 2039 /* 2040 * Once we unlock here, the zone cannot be grown anymore, thus if an 2041 * interrupt thread must allocate this early in boot, zone must be 2042 * pre-grown prior to start of deferred page initialization. 2043 */ 2044 pgdat_resize_unlock(pgdat, &flags); 2045 2046 /* Only the highest zone is deferred so find it */ 2047 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 2048 zone = pgdat->node_zones + zid; 2049 if (first_init_pfn < zone_end_pfn(zone)) 2050 break; 2051 } 2052 2053 /* If the zone is empty somebody else may have cleared out the zone */ 2054 if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, 2055 first_init_pfn)) 2056 goto zone_empty; 2057 2058 max_threads = deferred_page_init_max_threads(cpumask); 2059 2060 while (spfn < epfn) { 2061 unsigned long epfn_align = ALIGN(epfn, PAGES_PER_SECTION); 2062 struct padata_mt_job job = { 2063 .thread_fn = deferred_init_memmap_chunk, 2064 .fn_arg = zone, 2065 .start = spfn, 2066 .size = epfn_align - spfn, 2067 .align = PAGES_PER_SECTION, 2068 .min_chunk = PAGES_PER_SECTION, 2069 .max_threads = max_threads, 2070 }; 2071 2072 padata_do_multithreaded(&job); 2073 deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, 2074 epfn_align); 2075 } 2076 zone_empty: 2077 /* Sanity check that the next zone really is unpopulated */ 2078 WARN_ON(++zid < MAX_NR_ZONES && populated_zone(++zone)); 2079 2080 pr_info("node %d deferred pages initialised in %ums\n", 2081 pgdat->node_id, jiffies_to_msecs(jiffies - start)); 2082 2083 pgdat_init_report_one_done(); 2084 return 0; 2085 } 2086 2087 /* 2088 * If this zone has deferred pages, try to grow it by initializing enough 2089 * deferred pages to satisfy the allocation specified by order, rounded up to 2090 * the nearest PAGES_PER_SECTION boundary. So we're adding memory in increments 2091 * of SECTION_SIZE bytes by initializing struct pages in increments of 2092 * PAGES_PER_SECTION * sizeof(struct page) bytes. 2093 * 2094 * Return true when zone was grown, otherwise return false. We return true even 2095 * when we grow less than requested, to let the caller decide if there are 2096 * enough pages to satisfy the allocation. 2097 * 2098 * Note: We use noinline because this function is needed only during boot, and 2099 * it is called from a __ref function _deferred_grow_zone. This way we are 2100 * making sure that it is not inlined into permanent text section. 2101 */ 2102 static noinline bool __init 2103 deferred_grow_zone(struct zone *zone, unsigned int order) 2104 { 2105 unsigned long nr_pages_needed = ALIGN(1 << order, PAGES_PER_SECTION); 2106 pg_data_t *pgdat = zone->zone_pgdat; 2107 unsigned long first_deferred_pfn = pgdat->first_deferred_pfn; 2108 unsigned long spfn, epfn, flags; 2109 unsigned long nr_pages = 0; 2110 u64 i; 2111 2112 /* Only the last zone may have deferred pages */ 2113 if (zone_end_pfn(zone) != pgdat_end_pfn(pgdat)) 2114 return false; 2115 2116 pgdat_resize_lock(pgdat, &flags); 2117 2118 /* 2119 * If someone grew this zone while we were waiting for spinlock, return 2120 * true, as there might be enough pages already. 2121 */ 2122 if (first_deferred_pfn != pgdat->first_deferred_pfn) { 2123 pgdat_resize_unlock(pgdat, &flags); 2124 return true; 2125 } 2126 2127 /* If the zone is empty somebody else may have cleared out the zone */ 2128 if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, 2129 first_deferred_pfn)) { 2130 pgdat->first_deferred_pfn = ULONG_MAX; 2131 pgdat_resize_unlock(pgdat, &flags); 2132 /* Retry only once. */ 2133 return first_deferred_pfn != ULONG_MAX; 2134 } 2135 2136 /* 2137 * Initialize and free pages in MAX_ORDER sized increments so 2138 * that we can avoid introducing any issues with the buddy 2139 * allocator. 2140 */ 2141 while (spfn < epfn) { 2142 /* update our first deferred PFN for this section */ 2143 first_deferred_pfn = spfn; 2144 2145 nr_pages += deferred_init_maxorder(&i, zone, &spfn, &epfn); 2146 touch_nmi_watchdog(); 2147 2148 /* We should only stop along section boundaries */ 2149 if ((first_deferred_pfn ^ spfn) < PAGES_PER_SECTION) 2150 continue; 2151 2152 /* If our quota has been met we can stop here */ 2153 if (nr_pages >= nr_pages_needed) 2154 break; 2155 } 2156 2157 pgdat->first_deferred_pfn = spfn; 2158 pgdat_resize_unlock(pgdat, &flags); 2159 2160 return nr_pages > 0; 2161 } 2162 2163 /* 2164 * deferred_grow_zone() is __init, but it is called from 2165 * get_page_from_freelist() during early boot until deferred_pages permanently 2166 * disables this call. This is why we have refdata wrapper to avoid warning, 2167 * and to ensure that the function body gets unloaded. 2168 */ 2169 static bool __ref 2170 _deferred_grow_zone(struct zone *zone, unsigned int order) 2171 { 2172 return deferred_grow_zone(zone, order); 2173 } 2174 2175 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ 2176 2177 void __init page_alloc_init_late(void) 2178 { 2179 struct zone *zone; 2180 int nid; 2181 2182 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 2183 2184 /* There will be num_node_state(N_MEMORY) threads */ 2185 atomic_set(&pgdat_init_n_undone, num_node_state(N_MEMORY)); 2186 for_each_node_state(nid, N_MEMORY) { 2187 kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid); 2188 } 2189 2190 /* Block until all are initialised */ 2191 wait_for_completion(&pgdat_init_all_done_comp); 2192 2193 /* 2194 * We initialized the rest of the deferred pages. Permanently disable 2195 * on-demand struct page initialization. 2196 */ 2197 static_branch_disable(&deferred_pages); 2198 2199 /* Reinit limits that are based on free pages after the kernel is up */ 2200 files_maxfiles_init(); 2201 #endif 2202 2203 buffer_init(); 2204 2205 /* Discard memblock private memory */ 2206 memblock_discard(); 2207 2208 for_each_node_state(nid, N_MEMORY) 2209 shuffle_free_memory(NODE_DATA(nid)); 2210 2211 for_each_populated_zone(zone) 2212 set_zone_contiguous(zone); 2213 } 2214 2215 #ifdef CONFIG_CMA 2216 /* Free whole pageblock and set its migration type to MIGRATE_CMA. */ 2217 void __init init_cma_reserved_pageblock(struct page *page) 2218 { 2219 unsigned i = pageblock_nr_pages; 2220 struct page *p = page; 2221 2222 do { 2223 __ClearPageReserved(p); 2224 set_page_count(p, 0); 2225 } while (++p, --i); 2226 2227 set_pageblock_migratetype(page, MIGRATE_CMA); 2228 set_page_refcounted(page); 2229 __free_pages(page, pageblock_order); 2230 2231 adjust_managed_page_count(page, pageblock_nr_pages); 2232 page_zone(page)->cma_pages += pageblock_nr_pages; 2233 } 2234 #endif 2235 2236 /* 2237 * The order of subdivision here is critical for the IO subsystem. 2238 * Please do not alter this order without good reasons and regression 2239 * testing. Specifically, as large blocks of memory are subdivided, 2240 * the order in which smaller blocks are delivered depends on the order 2241 * they're subdivided in this function. This is the primary factor 2242 * influencing the order in which pages are delivered to the IO 2243 * subsystem according to empirical testing, and this is also justified 2244 * by considering the behavior of a buddy system containing a single 2245 * large block of memory acted on by a series of small allocations. 2246 * This behavior is a critical factor in sglist merging's success. 2247 * 2248 * -- nyc 2249 */ 2250 static inline void expand(struct zone *zone, struct page *page, 2251 int low, int high, int migratetype) 2252 { 2253 unsigned long size = 1 << high; 2254 2255 while (high > low) { 2256 high--; 2257 size >>= 1; 2258 VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]); 2259 2260 /* 2261 * Mark as guard pages (or page), that will allow to 2262 * merge back to allocator when buddy will be freed. 2263 * Corresponding page table entries will not be touched, 2264 * pages will stay not present in virtual address space 2265 */ 2266 if (set_page_guard(zone, &page[size], high, migratetype)) 2267 continue; 2268 2269 add_to_free_list(&page[size], zone, high, migratetype); 2270 set_buddy_order(&page[size], high); 2271 } 2272 } 2273 2274 static void check_new_page_bad(struct page *page) 2275 { 2276 if (unlikely(page->flags & __PG_HWPOISON)) { 2277 /* Don't complain about hwpoisoned pages */ 2278 page_mapcount_reset(page); /* remove PageBuddy */ 2279 return; 2280 } 2281 2282 bad_page(page, 2283 page_bad_reason(page, PAGE_FLAGS_CHECK_AT_PREP)); 2284 } 2285 2286 /* 2287 * This page is about to be returned from the page allocator 2288 */ 2289 static inline int check_new_page(struct page *page) 2290 { 2291 if (likely(page_expected_state(page, 2292 PAGE_FLAGS_CHECK_AT_PREP|__PG_HWPOISON))) 2293 return 0; 2294 2295 check_new_page_bad(page); 2296 return 1; 2297 } 2298 2299 static bool check_new_pages(struct page *page, unsigned int order) 2300 { 2301 int i; 2302 for (i = 0; i < (1 << order); i++) { 2303 struct page *p = page + i; 2304 2305 if (unlikely(check_new_page(p))) 2306 return true; 2307 } 2308 2309 return false; 2310 } 2311 2312 #ifdef CONFIG_DEBUG_VM 2313 /* 2314 * With DEBUG_VM enabled, order-0 pages are checked for expected state when 2315 * being allocated from pcp lists. With debug_pagealloc also enabled, they are 2316 * also checked when pcp lists are refilled from the free lists. 2317 */ 2318 static inline bool check_pcp_refill(struct page *page, unsigned int order) 2319 { 2320 if (debug_pagealloc_enabled_static()) 2321 return check_new_pages(page, order); 2322 else 2323 return false; 2324 } 2325 2326 static inline bool check_new_pcp(struct page *page, unsigned int order) 2327 { 2328 return check_new_pages(page, order); 2329 } 2330 #else 2331 /* 2332 * With DEBUG_VM disabled, free order-0 pages are checked for expected state 2333 * when pcp lists are being refilled from the free lists. With debug_pagealloc 2334 * enabled, they are also checked when being allocated from the pcp lists. 2335 */ 2336 static inline bool check_pcp_refill(struct page *page, unsigned int order) 2337 { 2338 return check_new_pages(page, order); 2339 } 2340 static inline bool check_new_pcp(struct page *page, unsigned int order) 2341 { 2342 if (debug_pagealloc_enabled_static()) 2343 return check_new_pages(page, order); 2344 else 2345 return false; 2346 } 2347 #endif /* CONFIG_DEBUG_VM */ 2348 2349 static inline bool should_skip_kasan_unpoison(gfp_t flags, bool init_tags) 2350 { 2351 /* Don't skip if a software KASAN mode is enabled. */ 2352 if (IS_ENABLED(CONFIG_KASAN_GENERIC) || 2353 IS_ENABLED(CONFIG_KASAN_SW_TAGS)) 2354 return false; 2355 2356 /* Skip, if hardware tag-based KASAN is not enabled. */ 2357 if (!kasan_hw_tags_enabled()) 2358 return true; 2359 2360 /* 2361 * With hardware tag-based KASAN enabled, skip if either: 2362 * 2363 * 1. Memory tags have already been cleared via tag_clear_highpage(). 2364 * 2. Skipping has been requested via __GFP_SKIP_KASAN_UNPOISON. 2365 */ 2366 return init_tags || (flags & __GFP_SKIP_KASAN_UNPOISON); 2367 } 2368 2369 static inline bool should_skip_init(gfp_t flags) 2370 { 2371 /* Don't skip, if hardware tag-based KASAN is not enabled. */ 2372 if (!kasan_hw_tags_enabled()) 2373 return false; 2374 2375 /* For hardware tag-based KASAN, skip if requested. */ 2376 return (flags & __GFP_SKIP_ZERO); 2377 } 2378 2379 inline void post_alloc_hook(struct page *page, unsigned int order, 2380 gfp_t gfp_flags) 2381 { 2382 bool init = !want_init_on_free() && want_init_on_alloc(gfp_flags) && 2383 !should_skip_init(gfp_flags); 2384 bool init_tags = init && (gfp_flags & __GFP_ZEROTAGS); 2385 2386 set_page_private(page, 0); 2387 set_page_refcounted(page); 2388 2389 arch_alloc_page(page, order); 2390 debug_pagealloc_map_pages(page, 1 << order); 2391 2392 /* 2393 * Page unpoisoning must happen before memory initialization. 2394 * Otherwise, the poison pattern will be overwritten for __GFP_ZERO 2395 * allocations and the page unpoisoning code will complain. 2396 */ 2397 kernel_unpoison_pages(page, 1 << order); 2398 2399 /* 2400 * As memory initialization might be integrated into KASAN, 2401 * KASAN unpoisoning and memory initializion code must be 2402 * kept together to avoid discrepancies in behavior. 2403 */ 2404 2405 /* 2406 * If memory tags should be zeroed (which happens only when memory 2407 * should be initialized as well). 2408 */ 2409 if (init_tags) { 2410 int i; 2411 2412 /* Initialize both memory and tags. */ 2413 for (i = 0; i != 1 << order; ++i) 2414 tag_clear_highpage(page + i); 2415 2416 /* Note that memory is already initialized by the loop above. */ 2417 init = false; 2418 } 2419 if (!should_skip_kasan_unpoison(gfp_flags, init_tags)) { 2420 /* Unpoison shadow memory or set memory tags. */ 2421 kasan_unpoison_pages(page, order, init); 2422 2423 /* Note that memory is already initialized by KASAN. */ 2424 if (kasan_has_integrated_init()) 2425 init = false; 2426 } 2427 /* If memory is still not initialized, do it now. */ 2428 if (init) 2429 kernel_init_free_pages(page, 1 << order); 2430 /* Propagate __GFP_SKIP_KASAN_POISON to page flags. */ 2431 if (kasan_hw_tags_enabled() && (gfp_flags & __GFP_SKIP_KASAN_POISON)) 2432 SetPageSkipKASanPoison(page); 2433 2434 set_page_owner(page, order, gfp_flags); 2435 page_table_check_alloc(page, order); 2436 } 2437 2438 static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags, 2439 unsigned int alloc_flags) 2440 { 2441 post_alloc_hook(page, order, gfp_flags); 2442 2443 if (order && (gfp_flags & __GFP_COMP)) 2444 prep_compound_page(page, order); 2445 2446 /* 2447 * page is set pfmemalloc when ALLOC_NO_WATERMARKS was necessary to 2448 * allocate the page. The expectation is that the caller is taking 2449 * steps that will free more memory. The caller should avoid the page 2450 * being used for !PFMEMALLOC purposes. 2451 */ 2452 if (alloc_flags & ALLOC_NO_WATERMARKS) 2453 set_page_pfmemalloc(page); 2454 else 2455 clear_page_pfmemalloc(page); 2456 } 2457 2458 /* 2459 * Go through the free lists for the given migratetype and remove 2460 * the smallest available page from the freelists 2461 */ 2462 static __always_inline 2463 struct page *__rmqueue_smallest(struct zone *zone, unsigned int order, 2464 int migratetype) 2465 { 2466 unsigned int current_order; 2467 struct free_area *area; 2468 struct page *page; 2469 2470 /* Find a page of the appropriate size in the preferred list */ 2471 for (current_order = order; current_order < MAX_ORDER; ++current_order) { 2472 area = &(zone->free_area[current_order]); 2473 page = get_page_from_free_area(area, migratetype); 2474 if (!page) 2475 continue; 2476 del_page_from_free_list(page, zone, current_order); 2477 expand(zone, page, order, current_order, migratetype); 2478 set_pcppage_migratetype(page, migratetype); 2479 return page; 2480 } 2481 2482 return NULL; 2483 } 2484 2485 2486 /* 2487 * This array describes the order lists are fallen back to when 2488 * the free lists for the desirable migrate type are depleted 2489 * 2490 * The other migratetypes do not have fallbacks. 2491 */ 2492 static int fallbacks[MIGRATE_TYPES][3] = { 2493 [MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE, MIGRATE_TYPES }, 2494 [MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_TYPES }, 2495 [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, MIGRATE_TYPES }, 2496 }; 2497 2498 #ifdef CONFIG_CMA 2499 static __always_inline struct page *__rmqueue_cma_fallback(struct zone *zone, 2500 unsigned int order) 2501 { 2502 return __rmqueue_smallest(zone, order, MIGRATE_CMA); 2503 } 2504 #else 2505 static inline struct page *__rmqueue_cma_fallback(struct zone *zone, 2506 unsigned int order) { return NULL; } 2507 #endif 2508 2509 /* 2510 * Move the free pages in a range to the freelist tail of the requested type. 2511 * Note that start_page and end_pages are not aligned on a pageblock 2512 * boundary. If alignment is required, use move_freepages_block() 2513 */ 2514 static int move_freepages(struct zone *zone, 2515 unsigned long start_pfn, unsigned long end_pfn, 2516 int migratetype, int *num_movable) 2517 { 2518 struct page *page; 2519 unsigned long pfn; 2520 unsigned int order; 2521 int pages_moved = 0; 2522 2523 for (pfn = start_pfn; pfn <= end_pfn;) { 2524 page = pfn_to_page(pfn); 2525 if (!PageBuddy(page)) { 2526 /* 2527 * We assume that pages that could be isolated for 2528 * migration are movable. But we don't actually try 2529 * isolating, as that would be expensive. 2530 */ 2531 if (num_movable && 2532 (PageLRU(page) || __PageMovable(page))) 2533 (*num_movable)++; 2534 pfn++; 2535 continue; 2536 } 2537 2538 /* Make sure we are not inadvertently changing nodes */ 2539 VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page); 2540 VM_BUG_ON_PAGE(page_zone(page) != zone, page); 2541 2542 order = buddy_order(page); 2543 move_to_free_list(page, zone, order, migratetype); 2544 pfn += 1 << order; 2545 pages_moved += 1 << order; 2546 } 2547 2548 return pages_moved; 2549 } 2550 2551 int move_freepages_block(struct zone *zone, struct page *page, 2552 int migratetype, int *num_movable) 2553 { 2554 unsigned long start_pfn, end_pfn, pfn; 2555 2556 if (num_movable) 2557 *num_movable = 0; 2558 2559 pfn = page_to_pfn(page); 2560 start_pfn = pfn & ~(pageblock_nr_pages - 1); 2561 end_pfn = start_pfn + pageblock_nr_pages - 1; 2562 2563 /* Do not cross zone boundaries */ 2564 if (!zone_spans_pfn(zone, start_pfn)) 2565 start_pfn = pfn; 2566 if (!zone_spans_pfn(zone, end_pfn)) 2567 return 0; 2568 2569 return move_freepages(zone, start_pfn, end_pfn, migratetype, 2570 num_movable); 2571 } 2572 2573 static void change_pageblock_range(struct page *pageblock_page, 2574 int start_order, int migratetype) 2575 { 2576 int nr_pageblocks = 1 << (start_order - pageblock_order); 2577 2578 while (nr_pageblocks--) { 2579 set_pageblock_migratetype(pageblock_page, migratetype); 2580 pageblock_page += pageblock_nr_pages; 2581 } 2582 } 2583 2584 /* 2585 * When we are falling back to another migratetype during allocation, try to 2586 * steal extra free pages from the same pageblocks to satisfy further 2587 * allocations, instead of polluting multiple pageblocks. 2588 * 2589 * If we are stealing a relatively large buddy page, it is likely there will 2590 * be more free pages in the pageblock, so try to steal them all. For 2591 * reclaimable and unmovable allocations, we steal regardless of page size, 2592 * as fragmentation caused by those allocations polluting movable pageblocks 2593 * is worse than movable allocations stealing from unmovable and reclaimable 2594 * pageblocks. 2595 */ 2596 static bool can_steal_fallback(unsigned int order, int start_mt) 2597 { 2598 /* 2599 * Leaving this order check is intended, although there is 2600 * relaxed order check in next check. The reason is that 2601 * we can actually steal whole pageblock if this condition met, 2602 * but, below check doesn't guarantee it and that is just heuristic 2603 * so could be changed anytime. 2604 */ 2605 if (order >= pageblock_order) 2606 return true; 2607 2608 if (order >= pageblock_order / 2 || 2609 start_mt == MIGRATE_RECLAIMABLE || 2610 start_mt == MIGRATE_UNMOVABLE || 2611 page_group_by_mobility_disabled) 2612 return true; 2613 2614 return false; 2615 } 2616 2617 static inline bool boost_watermark(struct zone *zone) 2618 { 2619 unsigned long max_boost; 2620 2621 if (!watermark_boost_factor) 2622 return false; 2623 /* 2624 * Don't bother in zones that are unlikely to produce results. 2625 * On small machines, including kdump capture kernels running 2626 * in a small area, boosting the watermark can cause an out of 2627 * memory situation immediately. 2628 */ 2629 if ((pageblock_nr_pages * 4) > zone_managed_pages(zone)) 2630 return false; 2631 2632 max_boost = mult_frac(zone->_watermark[WMARK_HIGH], 2633 watermark_boost_factor, 10000); 2634 2635 /* 2636 * high watermark may be uninitialised if fragmentation occurs 2637 * very early in boot so do not boost. We do not fall 2638 * through and boost by pageblock_nr_pages as failing 2639 * allocations that early means that reclaim is not going 2640 * to help and it may even be impossible to reclaim the 2641 * boosted watermark resulting in a hang. 2642 */ 2643 if (!max_boost) 2644 return false; 2645 2646 max_boost = max(pageblock_nr_pages, max_boost); 2647 2648 zone->watermark_boost = min(zone->watermark_boost + pageblock_nr_pages, 2649 max_boost); 2650 2651 return true; 2652 } 2653 2654 /* 2655 * This function implements actual steal behaviour. If order is large enough, 2656 * we can steal whole pageblock. If not, we first move freepages in this 2657 * pageblock to our migratetype and determine how many already-allocated pages 2658 * are there in the pageblock with a compatible migratetype. If at least half 2659 * of pages are free or compatible, we can change migratetype of the pageblock 2660 * itself, so pages freed in the future will be put on the correct free list. 2661 */ 2662 static void steal_suitable_fallback(struct zone *zone, struct page *page, 2663 unsigned int alloc_flags, int start_type, bool whole_block) 2664 { 2665 unsigned int current_order = buddy_order(page); 2666 int free_pages, movable_pages, alike_pages; 2667 int old_block_type; 2668 2669 old_block_type = get_pageblock_migratetype(page); 2670 2671 /* 2672 * This can happen due to races and we want to prevent broken 2673 * highatomic accounting. 2674 */ 2675 if (is_migrate_highatomic(old_block_type)) 2676 goto single_page; 2677 2678 /* Take ownership for orders >= pageblock_order */ 2679 if (current_order >= pageblock_order) { 2680 change_pageblock_range(page, current_order, start_type); 2681 goto single_page; 2682 } 2683 2684 /* 2685 * Boost watermarks to increase reclaim pressure to reduce the 2686 * likelihood of future fallbacks. Wake kswapd now as the node 2687 * may be balanced overall and kswapd will not wake naturally. 2688 */ 2689 if (boost_watermark(zone) && (alloc_flags & ALLOC_KSWAPD)) 2690 set_bit(ZONE_BOOSTED_WATERMARK, &zone->flags); 2691 2692 /* We are not allowed to try stealing from the whole block */ 2693 if (!whole_block) 2694 goto single_page; 2695 2696 free_pages = move_freepages_block(zone, page, start_type, 2697 &movable_pages); 2698 /* 2699 * Determine how many pages are compatible with our allocation. 2700 * For movable allocation, it's the number of movable pages which 2701 * we just obtained. For other types it's a bit more tricky. 2702 */ 2703 if (start_type == MIGRATE_MOVABLE) { 2704 alike_pages = movable_pages; 2705 } else { 2706 /* 2707 * If we are falling back a RECLAIMABLE or UNMOVABLE allocation 2708 * to MOVABLE pageblock, consider all non-movable pages as 2709 * compatible. If it's UNMOVABLE falling back to RECLAIMABLE or 2710 * vice versa, be conservative since we can't distinguish the 2711 * exact migratetype of non-movable pages. 2712 */ 2713 if (old_block_type == MIGRATE_MOVABLE) 2714 alike_pages = pageblock_nr_pages 2715 - (free_pages + movable_pages); 2716 else 2717 alike_pages = 0; 2718 } 2719 2720 /* moving whole block can fail due to zone boundary conditions */ 2721 if (!free_pages) 2722 goto single_page; 2723 2724 /* 2725 * If a sufficient number of pages in the block are either free or of 2726 * comparable migratability as our allocation, claim the whole block. 2727 */ 2728 if (free_pages + alike_pages >= (1 << (pageblock_order-1)) || 2729 page_group_by_mobility_disabled) 2730 set_pageblock_migratetype(page, start_type); 2731 2732 return; 2733 2734 single_page: 2735 move_to_free_list(page, zone, current_order, start_type); 2736 } 2737 2738 /* 2739 * Check whether there is a suitable fallback freepage with requested order. 2740 * If only_stealable is true, this function returns fallback_mt only if 2741 * we can steal other freepages all together. This would help to reduce 2742 * fragmentation due to mixed migratetype pages in one pageblock. 2743 */ 2744 int find_suitable_fallback(struct free_area *area, unsigned int order, 2745 int migratetype, bool only_stealable, bool *can_steal) 2746 { 2747 int i; 2748 int fallback_mt; 2749 2750 if (area->nr_free == 0) 2751 return -1; 2752 2753 *can_steal = false; 2754 for (i = 0;; i++) { 2755 fallback_mt = fallbacks[migratetype][i]; 2756 if (fallback_mt == MIGRATE_TYPES) 2757 break; 2758 2759 if (free_area_empty(area, fallback_mt)) 2760 continue; 2761 2762 if (can_steal_fallback(order, migratetype)) 2763 *can_steal = true; 2764 2765 if (!only_stealable) 2766 return fallback_mt; 2767 2768 if (*can_steal) 2769 return fallback_mt; 2770 } 2771 2772 return -1; 2773 } 2774 2775 /* 2776 * Reserve a pageblock for exclusive use of high-order atomic allocations if 2777 * there are no empty page blocks that contain a page with a suitable order 2778 */ 2779 static void reserve_highatomic_pageblock(struct page *page, struct zone *zone, 2780 unsigned int alloc_order) 2781 { 2782 int mt; 2783 unsigned long max_managed, flags; 2784 2785 /* 2786 * Limit the number reserved to 1 pageblock or roughly 1% of a zone. 2787 * Check is race-prone but harmless. 2788 */ 2789 max_managed = (zone_managed_pages(zone) / 100) + pageblock_nr_pages; 2790 if (zone->nr_reserved_highatomic >= max_managed) 2791 return; 2792 2793 spin_lock_irqsave(&zone->lock, flags); 2794 2795 /* Recheck the nr_reserved_highatomic limit under the lock */ 2796 if (zone->nr_reserved_highatomic >= max_managed) 2797 goto out_unlock; 2798 2799 /* Yoink! */ 2800 mt = get_pageblock_migratetype(page); 2801 /* Only reserve normal pageblocks (i.e., they can merge with others) */ 2802 if (migratetype_is_mergeable(mt)) { 2803 zone->nr_reserved_highatomic += pageblock_nr_pages; 2804 set_pageblock_migratetype(page, MIGRATE_HIGHATOMIC); 2805 move_freepages_block(zone, page, MIGRATE_HIGHATOMIC, NULL); 2806 } 2807 2808 out_unlock: 2809 spin_unlock_irqrestore(&zone->lock, flags); 2810 } 2811 2812 /* 2813 * Used when an allocation is about to fail under memory pressure. This 2814 * potentially hurts the reliability of high-order allocations when under 2815 * intense memory pressure but failed atomic allocations should be easier 2816 * to recover from than an OOM. 2817 * 2818 * If @force is true, try to unreserve a pageblock even though highatomic 2819 * pageblock is exhausted. 2820 */ 2821 static bool unreserve_highatomic_pageblock(const struct alloc_context *ac, 2822 bool force) 2823 { 2824 struct zonelist *zonelist = ac->zonelist; 2825 unsigned long flags; 2826 struct zoneref *z; 2827 struct zone *zone; 2828 struct page *page; 2829 int order; 2830 bool ret; 2831 2832 for_each_zone_zonelist_nodemask(zone, z, zonelist, ac->highest_zoneidx, 2833 ac->nodemask) { 2834 /* 2835 * Preserve at least one pageblock unless memory pressure 2836 * is really high. 2837 */ 2838 if (!force && zone->nr_reserved_highatomic <= 2839 pageblock_nr_pages) 2840 continue; 2841 2842 spin_lock_irqsave(&zone->lock, flags); 2843 for (order = 0; order < MAX_ORDER; order++) { 2844 struct free_area *area = &(zone->free_area[order]); 2845 2846 page = get_page_from_free_area(area, MIGRATE_HIGHATOMIC); 2847 if (!page) 2848 continue; 2849 2850 /* 2851 * In page freeing path, migratetype change is racy so 2852 * we can counter several free pages in a pageblock 2853 * in this loop although we changed the pageblock type 2854 * from highatomic to ac->migratetype. So we should 2855 * adjust the count once. 2856 */ 2857 if (is_migrate_highatomic_page(page)) { 2858 /* 2859 * It should never happen but changes to 2860 * locking could inadvertently allow a per-cpu 2861 * drain to add pages to MIGRATE_HIGHATOMIC 2862 * while unreserving so be safe and watch for 2863 * underflows. 2864 */ 2865 zone->nr_reserved_highatomic -= min( 2866 pageblock_nr_pages, 2867 zone->nr_reserved_highatomic); 2868 } 2869 2870 /* 2871 * Convert to ac->migratetype and avoid the normal 2872 * pageblock stealing heuristics. Minimally, the caller 2873 * is doing the work and needs the pages. More 2874 * importantly, if the block was always converted to 2875 * MIGRATE_UNMOVABLE or another type then the number 2876 * of pageblocks that cannot be completely freed 2877 * may increase. 2878 */ 2879 set_pageblock_migratetype(page, ac->migratetype); 2880 ret = move_freepages_block(zone, page, ac->migratetype, 2881 NULL); 2882 if (ret) { 2883 spin_unlock_irqrestore(&zone->lock, flags); 2884 return ret; 2885 } 2886 } 2887 spin_unlock_irqrestore(&zone->lock, flags); 2888 } 2889 2890 return false; 2891 } 2892 2893 /* 2894 * Try finding a free buddy page on the fallback list and put it on the free 2895 * list of requested migratetype, possibly along with other pages from the same 2896 * block, depending on fragmentation avoidance heuristics. Returns true if 2897 * fallback was found so that __rmqueue_smallest() can grab it. 2898 * 2899 * The use of signed ints for order and current_order is a deliberate 2900 * deviation from the rest of this file, to make the for loop 2901 * condition simpler. 2902 */ 2903 static __always_inline bool 2904 __rmqueue_fallback(struct zone *zone, int order, int start_migratetype, 2905 unsigned int alloc_flags) 2906 { 2907 struct free_area *area; 2908 int current_order; 2909 int min_order = order; 2910 struct page *page; 2911 int fallback_mt; 2912 bool can_steal; 2913 2914 /* 2915 * Do not steal pages from freelists belonging to other pageblocks 2916 * i.e. orders < pageblock_order. If there are no local zones free, 2917 * the zonelists will be reiterated without ALLOC_NOFRAGMENT. 2918 */ 2919 if (alloc_flags & ALLOC_NOFRAGMENT) 2920 min_order = pageblock_order; 2921 2922 /* 2923 * Find the largest available free page in the other list. This roughly 2924 * approximates finding the pageblock with the most free pages, which 2925 * would be too costly to do exactly. 2926 */ 2927 for (current_order = MAX_ORDER - 1; current_order >= min_order; 2928 --current_order) { 2929 area = &(zone->free_area[current_order]); 2930 fallback_mt = find_suitable_fallback(area, current_order, 2931 start_migratetype, false, &can_steal); 2932 if (fallback_mt == -1) 2933 continue; 2934 2935 /* 2936 * We cannot steal all free pages from the pageblock and the 2937 * requested migratetype is movable. In that case it's better to 2938 * steal and split the smallest available page instead of the 2939 * largest available page, because even if the next movable 2940 * allocation falls back into a different pageblock than this 2941 * one, it won't cause permanent fragmentation. 2942 */ 2943 if (!can_steal && start_migratetype == MIGRATE_MOVABLE 2944 && current_order > order) 2945 goto find_smallest; 2946 2947 goto do_steal; 2948 } 2949 2950 return false; 2951 2952 find_smallest: 2953 for (current_order = order; current_order < MAX_ORDER; 2954 current_order++) { 2955 area = &(zone->free_area[current_order]); 2956 fallback_mt = find_suitable_fallback(area, current_order, 2957 start_migratetype, false, &can_steal); 2958 if (fallback_mt != -1) 2959 break; 2960 } 2961 2962 /* 2963 * This should not happen - we already found a suitable fallback 2964 * when looking for the largest page. 2965 */ 2966 VM_BUG_ON(current_order == MAX_ORDER); 2967 2968 do_steal: 2969 page = get_page_from_free_area(area, fallback_mt); 2970 2971 steal_suitable_fallback(zone, page, alloc_flags, start_migratetype, 2972 can_steal); 2973 2974 trace_mm_page_alloc_extfrag(page, order, current_order, 2975 start_migratetype, fallback_mt); 2976 2977 return true; 2978 2979 } 2980 2981 /* 2982 * Do the hard work of removing an element from the buddy allocator. 2983 * Call me with the zone->lock already held. 2984 */ 2985 static __always_inline struct page * 2986 __rmqueue(struct zone *zone, unsigned int order, int migratetype, 2987 unsigned int alloc_flags) 2988 { 2989 struct page *page; 2990 2991 if (IS_ENABLED(CONFIG_CMA)) { 2992 /* 2993 * Balance movable allocations between regular and CMA areas by 2994 * allocating from CMA when over half of the zone's free memory 2995 * is in the CMA area. 2996 */ 2997 if (alloc_flags & ALLOC_CMA && 2998 zone_page_state(zone, NR_FREE_CMA_PAGES) > 2999 zone_page_state(zone, NR_FREE_PAGES) / 2) { 3000 page = __rmqueue_cma_fallback(zone, order); 3001 if (page) 3002 goto out; 3003 } 3004 } 3005 retry: 3006 page = __rmqueue_smallest(zone, order, migratetype); 3007 if (unlikely(!page)) { 3008 if (alloc_flags & ALLOC_CMA) 3009 page = __rmqueue_cma_fallback(zone, order); 3010 3011 if (!page && __rmqueue_fallback(zone, order, migratetype, 3012 alloc_flags)) 3013 goto retry; 3014 } 3015 out: 3016 if (page) 3017 trace_mm_page_alloc_zone_locked(page, order, migratetype); 3018 return page; 3019 } 3020 3021 /* 3022 * Obtain a specified number of elements from the buddy allocator, all under 3023 * a single hold of the lock, for efficiency. Add them to the supplied list. 3024 * Returns the number of new pages which were placed at *list. 3025 */ 3026 static int rmqueue_bulk(struct zone *zone, unsigned int order, 3027 unsigned long count, struct list_head *list, 3028 int migratetype, unsigned int alloc_flags) 3029 { 3030 int i, allocated = 0; 3031 3032 /* 3033 * local_lock_irq held so equivalent to spin_lock_irqsave for 3034 * both PREEMPT_RT and non-PREEMPT_RT configurations. 3035 */ 3036 spin_lock(&zone->lock); 3037 for (i = 0; i < count; ++i) { 3038 struct page *page = __rmqueue(zone, order, migratetype, 3039 alloc_flags); 3040 if (unlikely(page == NULL)) 3041 break; 3042 3043 if (unlikely(check_pcp_refill(page, order))) 3044 continue; 3045 3046 /* 3047 * Split buddy pages returned by expand() are received here in 3048 * physical page order. The page is added to the tail of 3049 * caller's list. From the callers perspective, the linked list 3050 * is ordered by page number under some conditions. This is 3051 * useful for IO devices that can forward direction from the 3052 * head, thus also in the physical page order. This is useful 3053 * for IO devices that can merge IO requests if the physical 3054 * pages are ordered properly. 3055 */ 3056 list_add_tail(&page->lru, list); 3057 allocated++; 3058 if (is_migrate_cma(get_pcppage_migratetype(page))) 3059 __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, 3060 -(1 << order)); 3061 } 3062 3063 /* 3064 * i pages were removed from the buddy list even if some leak due 3065 * to check_pcp_refill failing so adjust NR_FREE_PAGES based 3066 * on i. Do not confuse with 'allocated' which is the number of 3067 * pages added to the pcp list. 3068 */ 3069 __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order)); 3070 spin_unlock(&zone->lock); 3071 return allocated; 3072 } 3073 3074 #ifdef CONFIG_NUMA 3075 /* 3076 * Called from the vmstat counter updater to drain pagesets of this 3077 * currently executing processor on remote nodes after they have 3078 * expired. 3079 * 3080 * Note that this function must be called with the thread pinned to 3081 * a single processor. 3082 */ 3083 void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp) 3084 { 3085 unsigned long flags; 3086 int to_drain, batch; 3087 3088 local_lock_irqsave(&pagesets.lock, flags); 3089 batch = READ_ONCE(pcp->batch); 3090 to_drain = min(pcp->count, batch); 3091 if (to_drain > 0) 3092 free_pcppages_bulk(zone, to_drain, pcp, 0); 3093 local_unlock_irqrestore(&pagesets.lock, flags); 3094 } 3095 #endif 3096 3097 /* 3098 * Drain pcplists of the indicated processor and zone. 3099 * 3100 * The processor must either be the current processor and the 3101 * thread pinned to the current processor or a processor that 3102 * is not online. 3103 */ 3104 static void drain_pages_zone(unsigned int cpu, struct zone *zone) 3105 { 3106 unsigned long flags; 3107 struct per_cpu_pages *pcp; 3108 3109 local_lock_irqsave(&pagesets.lock, flags); 3110 3111 pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); 3112 if (pcp->count) 3113 free_pcppages_bulk(zone, pcp->count, pcp, 0); 3114 3115 local_unlock_irqrestore(&pagesets.lock, flags); 3116 } 3117 3118 /* 3119 * Drain pcplists of all zones on the indicated processor. 3120 * 3121 * The processor must either be the current processor and the 3122 * thread pinned to the current processor or a processor that 3123 * is not online. 3124 */ 3125 static void drain_pages(unsigned int cpu) 3126 { 3127 struct zone *zone; 3128 3129 for_each_populated_zone(zone) { 3130 drain_pages_zone(cpu, zone); 3131 } 3132 } 3133 3134 /* 3135 * Spill all of this CPU's per-cpu pages back into the buddy allocator. 3136 * 3137 * The CPU has to be pinned. When zone parameter is non-NULL, spill just 3138 * the single zone's pages. 3139 */ 3140 void drain_local_pages(struct zone *zone) 3141 { 3142 int cpu = smp_processor_id(); 3143 3144 if (zone) 3145 drain_pages_zone(cpu, zone); 3146 else 3147 drain_pages(cpu); 3148 } 3149 3150 static void drain_local_pages_wq(struct work_struct *work) 3151 { 3152 struct pcpu_drain *drain; 3153 3154 drain = container_of(work, struct pcpu_drain, work); 3155 3156 /* 3157 * drain_all_pages doesn't use proper cpu hotplug protection so 3158 * we can race with cpu offline when the WQ can move this from 3159 * a cpu pinned worker to an unbound one. We can operate on a different 3160 * cpu which is alright but we also have to make sure to not move to 3161 * a different one. 3162 */ 3163 migrate_disable(); 3164 drain_local_pages(drain->zone); 3165 migrate_enable(); 3166 } 3167 3168 /* 3169 * The implementation of drain_all_pages(), exposing an extra parameter to 3170 * drain on all cpus. 3171 * 3172 * drain_all_pages() is optimized to only execute on cpus where pcplists are 3173 * not empty. The check for non-emptiness can however race with a free to 3174 * pcplist that has not yet increased the pcp->count from 0 to 1. Callers 3175 * that need the guarantee that every CPU has drained can disable the 3176 * optimizing racy check. 3177 */ 3178 static void __drain_all_pages(struct zone *zone, bool force_all_cpus) 3179 { 3180 int cpu; 3181 3182 /* 3183 * Allocate in the BSS so we won't require allocation in 3184 * direct reclaim path for CONFIG_CPUMASK_OFFSTACK=y 3185 */ 3186 static cpumask_t cpus_with_pcps; 3187 3188 /* 3189 * Make sure nobody triggers this path before mm_percpu_wq is fully 3190 * initialized. 3191 */ 3192 if (WARN_ON_ONCE(!mm_percpu_wq)) 3193 return; 3194 3195 /* 3196 * Do not drain if one is already in progress unless it's specific to 3197 * a zone. Such callers are primarily CMA and memory hotplug and need 3198 * the drain to be complete when the call returns. 3199 */ 3200 if (unlikely(!mutex_trylock(&pcpu_drain_mutex))) { 3201 if (!zone) 3202 return; 3203 mutex_lock(&pcpu_drain_mutex); 3204 } 3205 3206 /* 3207 * We don't care about racing with CPU hotplug event 3208 * as offline notification will cause the notified 3209 * cpu to drain that CPU pcps and on_each_cpu_mask 3210 * disables preemption as part of its processing 3211 */ 3212 for_each_online_cpu(cpu) { 3213 struct per_cpu_pages *pcp; 3214 struct zone *z; 3215 bool has_pcps = false; 3216 3217 if (force_all_cpus) { 3218 /* 3219 * The pcp.count check is racy, some callers need a 3220 * guarantee that no cpu is missed. 3221 */ 3222 has_pcps = true; 3223 } else if (zone) { 3224 pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); 3225 if (pcp->count) 3226 has_pcps = true; 3227 } else { 3228 for_each_populated_zone(z) { 3229 pcp = per_cpu_ptr(z->per_cpu_pageset, cpu); 3230 if (pcp->count) { 3231 has_pcps = true; 3232 break; 3233 } 3234 } 3235 } 3236 3237 if (has_pcps) 3238 cpumask_set_cpu(cpu, &cpus_with_pcps); 3239 else 3240 cpumask_clear_cpu(cpu, &cpus_with_pcps); 3241 } 3242 3243 for_each_cpu(cpu, &cpus_with_pcps) { 3244 struct pcpu_drain *drain = per_cpu_ptr(&pcpu_drain, cpu); 3245 3246 drain->zone = zone; 3247 INIT_WORK(&drain->work, drain_local_pages_wq); 3248 queue_work_on(cpu, mm_percpu_wq, &drain->work); 3249 } 3250 for_each_cpu(cpu, &cpus_with_pcps) 3251 flush_work(&per_cpu_ptr(&pcpu_drain, cpu)->work); 3252 3253 mutex_unlock(&pcpu_drain_mutex); 3254 } 3255 3256 /* 3257 * Spill all the per-cpu pages from all CPUs back into the buddy allocator. 3258 * 3259 * When zone parameter is non-NULL, spill just the single zone's pages. 3260 * 3261 * Note that this can be extremely slow as the draining happens in a workqueue. 3262 */ 3263 void drain_all_pages(struct zone *zone) 3264 { 3265 __drain_all_pages(zone, false); 3266 } 3267 3268 #ifdef CONFIG_HIBERNATION 3269 3270 /* 3271 * Touch the watchdog for every WD_PAGE_COUNT pages. 3272 */ 3273 #define WD_PAGE_COUNT (128*1024) 3274 3275 void mark_free_pages(struct zone *zone) 3276 { 3277 unsigned long pfn, max_zone_pfn, page_count = WD_PAGE_COUNT; 3278 unsigned long flags; 3279 unsigned int order, t; 3280 struct page *page; 3281 3282 if (zone_is_empty(zone)) 3283 return; 3284 3285 spin_lock_irqsave(&zone->lock, flags); 3286 3287 max_zone_pfn = zone_end_pfn(zone); 3288 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) 3289 if (pfn_valid(pfn)) { 3290 page = pfn_to_page(pfn); 3291 3292 if (!--page_count) { 3293 touch_nmi_watchdog(); 3294 page_count = WD_PAGE_COUNT; 3295 } 3296 3297 if (page_zone(page) != zone) 3298 continue; 3299 3300 if (!swsusp_page_is_forbidden(page)) 3301 swsusp_unset_page_free(page); 3302 } 3303 3304 for_each_migratetype_order(order, t) { 3305 list_for_each_entry(page, 3306 &zone->free_area[order].free_list[t], lru) { 3307 unsigned long i; 3308 3309 pfn = page_to_pfn(page); 3310 for (i = 0; i < (1UL << order); i++) { 3311 if (!--page_count) { 3312 touch_nmi_watchdog(); 3313 page_count = WD_PAGE_COUNT; 3314 } 3315 swsusp_set_page_free(pfn_to_page(pfn + i)); 3316 } 3317 } 3318 } 3319 spin_unlock_irqrestore(&zone->lock, flags); 3320 } 3321 #endif /* CONFIG_PM */ 3322 3323 static bool free_unref_page_prepare(struct page *page, unsigned long pfn, 3324 unsigned int order) 3325 { 3326 int migratetype; 3327 3328 if (!free_pcp_prepare(page, order)) 3329 return false; 3330 3331 migratetype = get_pfnblock_migratetype(page, pfn); 3332 set_pcppage_migratetype(page, migratetype); 3333 return true; 3334 } 3335 3336 static int nr_pcp_free(struct per_cpu_pages *pcp, int high, int batch, 3337 bool free_high) 3338 { 3339 int min_nr_free, max_nr_free; 3340 3341 /* Free everything if batch freeing high-order pages. */ 3342 if (unlikely(free_high)) 3343 return pcp->count; 3344 3345 /* Check for PCP disabled or boot pageset */ 3346 if (unlikely(high < batch)) 3347 return 1; 3348 3349 /* Leave at least pcp->batch pages on the list */ 3350 min_nr_free = batch; 3351 max_nr_free = high - batch; 3352 3353 /* 3354 * Double the number of pages freed each time there is subsequent 3355 * freeing of pages without any allocation. 3356 */ 3357 batch <<= pcp->free_factor; 3358 if (batch < max_nr_free) 3359 pcp->free_factor++; 3360 batch = clamp(batch, min_nr_free, max_nr_free); 3361 3362 return batch; 3363 } 3364 3365 static int nr_pcp_high(struct per_cpu_pages *pcp, struct zone *zone, 3366 bool free_high) 3367 { 3368 int high = READ_ONCE(pcp->high); 3369 3370 if (unlikely(!high || free_high)) 3371 return 0; 3372 3373 if (!test_bit(ZONE_RECLAIM_ACTIVE, &zone->flags)) 3374 return high; 3375 3376 /* 3377 * If reclaim is active, limit the number of pages that can be 3378 * stored on pcp lists 3379 */ 3380 return min(READ_ONCE(pcp->batch) << 2, high); 3381 } 3382 3383 static void free_unref_page_commit(struct page *page, int migratetype, 3384 unsigned int order) 3385 { 3386 struct zone *zone = page_zone(page); 3387 struct per_cpu_pages *pcp; 3388 int high; 3389 int pindex; 3390 bool free_high; 3391 3392 __count_vm_event(PGFREE); 3393 pcp = this_cpu_ptr(zone->per_cpu_pageset); 3394 pindex = order_to_pindex(migratetype, order); 3395 list_add(&page->lru, &pcp->lists[pindex]); 3396 pcp->count += 1 << order; 3397 3398 /* 3399 * As high-order pages other than THP's stored on PCP can contribute 3400 * to fragmentation, limit the number stored when PCP is heavily 3401 * freeing without allocation. The remainder after bulk freeing 3402 * stops will be drained from vmstat refresh context. 3403 */ 3404 free_high = (pcp->free_factor && order && order <= PAGE_ALLOC_COSTLY_ORDER); 3405 3406 high = nr_pcp_high(pcp, zone, free_high); 3407 if (pcp->count >= high) { 3408 int batch = READ_ONCE(pcp->batch); 3409 3410 free_pcppages_bulk(zone, nr_pcp_free(pcp, high, batch, free_high), pcp, pindex); 3411 } 3412 } 3413 3414 /* 3415 * Free a pcp page 3416 */ 3417 void free_unref_page(struct page *page, unsigned int order) 3418 { 3419 unsigned long flags; 3420 unsigned long pfn = page_to_pfn(page); 3421 int migratetype; 3422 3423 if (!free_unref_page_prepare(page, pfn, order)) 3424 return; 3425 3426 /* 3427 * We only track unmovable, reclaimable and movable on pcp lists. 3428 * Place ISOLATE pages on the isolated list because they are being 3429 * offlined but treat HIGHATOMIC as movable pages so we can get those 3430 * areas back if necessary. Otherwise, we may have to free 3431 * excessively into the page allocator 3432 */ 3433 migratetype = get_pcppage_migratetype(page); 3434 if (unlikely(migratetype >= MIGRATE_PCPTYPES)) { 3435 if (unlikely(is_migrate_isolate(migratetype))) { 3436 free_one_page(page_zone(page), page, pfn, order, migratetype, FPI_NONE); 3437 return; 3438 } 3439 migratetype = MIGRATE_MOVABLE; 3440 } 3441 3442 local_lock_irqsave(&pagesets.lock, flags); 3443 free_unref_page_commit(page, migratetype, order); 3444 local_unlock_irqrestore(&pagesets.lock, flags); 3445 } 3446 3447 /* 3448 * Free a list of 0-order pages 3449 */ 3450 void free_unref_page_list(struct list_head *list) 3451 { 3452 struct page *page, *next; 3453 unsigned long flags; 3454 int batch_count = 0; 3455 int migratetype; 3456 3457 /* Prepare pages for freeing */ 3458 list_for_each_entry_safe(page, next, list, lru) { 3459 unsigned long pfn = page_to_pfn(page); 3460 if (!free_unref_page_prepare(page, pfn, 0)) { 3461 list_del(&page->lru); 3462 continue; 3463 } 3464 3465 /* 3466 * Free isolated pages directly to the allocator, see 3467 * comment in free_unref_page. 3468 */ 3469 migratetype = get_pcppage_migratetype(page); 3470 if (unlikely(is_migrate_isolate(migratetype))) { 3471 list_del(&page->lru); 3472 free_one_page(page_zone(page), page, pfn, 0, migratetype, FPI_NONE); 3473 continue; 3474 } 3475 } 3476 3477 local_lock_irqsave(&pagesets.lock, flags); 3478 list_for_each_entry_safe(page, next, list, lru) { 3479 /* 3480 * Non-isolated types over MIGRATE_PCPTYPES get added 3481 * to the MIGRATE_MOVABLE pcp list. 3482 */ 3483 migratetype = get_pcppage_migratetype(page); 3484 if (unlikely(migratetype >= MIGRATE_PCPTYPES)) 3485 migratetype = MIGRATE_MOVABLE; 3486 3487 trace_mm_page_free_batched(page); 3488 free_unref_page_commit(page, migratetype, 0); 3489 3490 /* 3491 * Guard against excessive IRQ disabled times when we get 3492 * a large list of pages to free. 3493 */ 3494 if (++batch_count == SWAP_CLUSTER_MAX) { 3495 local_unlock_irqrestore(&pagesets.lock, flags); 3496 batch_count = 0; 3497 local_lock_irqsave(&pagesets.lock, flags); 3498 } 3499 } 3500 local_unlock_irqrestore(&pagesets.lock, flags); 3501 } 3502 3503 /* 3504 * split_page takes a non-compound higher-order page, and splits it into 3505 * n (1<<order) sub-pages: page[0..n] 3506 * Each sub-page must be freed individually. 3507 * 3508 * Note: this is probably too low level an operation for use in drivers. 3509 * Please consult with lkml before using this in your driver. 3510 */ 3511 void split_page(struct page *page, unsigned int order) 3512 { 3513 int i; 3514 3515 VM_BUG_ON_PAGE(PageCompound(page), page); 3516 VM_BUG_ON_PAGE(!page_count(page), page); 3517 3518 for (i = 1; i < (1 << order); i++) 3519 set_page_refcounted(page + i); 3520 split_page_owner(page, 1 << order); 3521 split_page_memcg(page, 1 << order); 3522 } 3523 EXPORT_SYMBOL_GPL(split_page); 3524 3525 int __isolate_free_page(struct page *page, unsigned int order) 3526 { 3527 unsigned long watermark; 3528 struct zone *zone; 3529 int mt; 3530 3531 BUG_ON(!PageBuddy(page)); 3532 3533 zone = page_zone(page); 3534 mt = get_pageblock_migratetype(page); 3535 3536 if (!is_migrate_isolate(mt)) { 3537 /* 3538 * Obey watermarks as if the page was being allocated. We can 3539 * emulate a high-order watermark check with a raised order-0 3540 * watermark, because we already know our high-order page 3541 * exists. 3542 */ 3543 watermark = zone->_watermark[WMARK_MIN] + (1UL << order); 3544 if (!zone_watermark_ok(zone, 0, watermark, 0, ALLOC_CMA)) 3545 return 0; 3546 3547 __mod_zone_freepage_state(zone, -(1UL << order), mt); 3548 } 3549 3550 /* Remove page from free list */ 3551 3552 del_page_from_free_list(page, zone, order); 3553 3554 /* 3555 * Set the pageblock if the isolated page is at least half of a 3556 * pageblock 3557 */ 3558 if (order >= pageblock_order - 1) { 3559 struct page *endpage = page + (1 << order) - 1; 3560 for (; page < endpage; page += pageblock_nr_pages) { 3561 int mt = get_pageblock_migratetype(page); 3562 /* 3563 * Only change normal pageblocks (i.e., they can merge 3564 * with others) 3565 */ 3566 if (migratetype_is_mergeable(mt)) 3567 set_pageblock_migratetype(page, 3568 MIGRATE_MOVABLE); 3569 } 3570 } 3571 3572 3573 return 1UL << order; 3574 } 3575 3576 /** 3577 * __putback_isolated_page - Return a now-isolated page back where we got it 3578 * @page: Page that was isolated 3579 * @order: Order of the isolated page 3580 * @mt: The page's pageblock's migratetype 3581 * 3582 * This function is meant to return a page pulled from the free lists via 3583 * __isolate_free_page back to the free lists they were pulled from. 3584 */ 3585 void __putback_isolated_page(struct page *page, unsigned int order, int mt) 3586 { 3587 struct zone *zone = page_zone(page); 3588 3589 /* zone lock should be held when this function is called */ 3590 lockdep_assert_held(&zone->lock); 3591 3592 /* Return isolated page to tail of freelist. */ 3593 __free_one_page(page, page_to_pfn(page), zone, order, mt, 3594 FPI_SKIP_REPORT_NOTIFY | FPI_TO_TAIL); 3595 } 3596 3597 /* 3598 * Update NUMA hit/miss statistics 3599 * 3600 * Must be called with interrupts disabled. 3601 */ 3602 static inline void zone_statistics(struct zone *preferred_zone, struct zone *z, 3603 long nr_account) 3604 { 3605 #ifdef CONFIG_NUMA 3606 enum numa_stat_item local_stat = NUMA_LOCAL; 3607 3608 /* skip numa counters update if numa stats is disabled */ 3609 if (!static_branch_likely(&vm_numa_stat_key)) 3610 return; 3611 3612 if (zone_to_nid(z) != numa_node_id()) 3613 local_stat = NUMA_OTHER; 3614 3615 if (zone_to_nid(z) == zone_to_nid(preferred_zone)) 3616 __count_numa_events(z, NUMA_HIT, nr_account); 3617 else { 3618 __count_numa_events(z, NUMA_MISS, nr_account); 3619 __count_numa_events(preferred_zone, NUMA_FOREIGN, nr_account); 3620 } 3621 __count_numa_events(z, local_stat, nr_account); 3622 #endif 3623 } 3624 3625 /* Remove page from the per-cpu list, caller must protect the list */ 3626 static inline 3627 struct page *__rmqueue_pcplist(struct zone *zone, unsigned int order, 3628 int migratetype, 3629 unsigned int alloc_flags, 3630 struct per_cpu_pages *pcp, 3631 struct list_head *list) 3632 { 3633 struct page *page; 3634 3635 do { 3636 if (list_empty(list)) { 3637 int batch = READ_ONCE(pcp->batch); 3638 int alloced; 3639 3640 /* 3641 * Scale batch relative to order if batch implies 3642 * free pages can be stored on the PCP. Batch can 3643 * be 1 for small zones or for boot pagesets which 3644 * should never store free pages as the pages may 3645 * belong to arbitrary zones. 3646 */ 3647 if (batch > 1) 3648 batch = max(batch >> order, 2); 3649 alloced = rmqueue_bulk(zone, order, 3650 batch, list, 3651 migratetype, alloc_flags); 3652 3653 pcp->count += alloced << order; 3654 if (unlikely(list_empty(list))) 3655 return NULL; 3656 } 3657 3658 page = list_first_entry(list, struct page, lru); 3659 list_del(&page->lru); 3660 pcp->count -= 1 << order; 3661 } while (check_new_pcp(page, order)); 3662 3663 return page; 3664 } 3665 3666 /* Lock and remove page from the per-cpu list */ 3667 static struct page *rmqueue_pcplist(struct zone *preferred_zone, 3668 struct zone *zone, unsigned int order, 3669 gfp_t gfp_flags, int migratetype, 3670 unsigned int alloc_flags) 3671 { 3672 struct per_cpu_pages *pcp; 3673 struct list_head *list; 3674 struct page *page; 3675 unsigned long flags; 3676 3677 local_lock_irqsave(&pagesets.lock, flags); 3678 3679 /* 3680 * On allocation, reduce the number of pages that are batch freed. 3681 * See nr_pcp_free() where free_factor is increased for subsequent 3682 * frees. 3683 */ 3684 pcp = this_cpu_ptr(zone->per_cpu_pageset); 3685 pcp->free_factor >>= 1; 3686 list = &pcp->lists[order_to_pindex(migratetype, order)]; 3687 page = __rmqueue_pcplist(zone, order, migratetype, alloc_flags, pcp, list); 3688 local_unlock_irqrestore(&pagesets.lock, flags); 3689 if (page) { 3690 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1); 3691 zone_statistics(preferred_zone, zone, 1); 3692 } 3693 return page; 3694 } 3695 3696 /* 3697 * Allocate a page from the given zone. Use pcplists for order-0 allocations. 3698 */ 3699 static inline 3700 struct page *rmqueue(struct zone *preferred_zone, 3701 struct zone *zone, unsigned int order, 3702 gfp_t gfp_flags, unsigned int alloc_flags, 3703 int migratetype) 3704 { 3705 unsigned long flags; 3706 struct page *page; 3707 3708 if (likely(pcp_allowed_order(order))) { 3709 /* 3710 * MIGRATE_MOVABLE pcplist could have the pages on CMA area and 3711 * we need to skip it when CMA area isn't allowed. 3712 */ 3713 if (!IS_ENABLED(CONFIG_CMA) || alloc_flags & ALLOC_CMA || 3714 migratetype != MIGRATE_MOVABLE) { 3715 page = rmqueue_pcplist(preferred_zone, zone, order, 3716 gfp_flags, migratetype, alloc_flags); 3717 goto out; 3718 } 3719 } 3720 3721 /* 3722 * We most definitely don't want callers attempting to 3723 * allocate greater than order-1 page units with __GFP_NOFAIL. 3724 */ 3725 WARN_ON_ONCE((gfp_flags & __GFP_NOFAIL) && (order > 1)); 3726 3727 do { 3728 page = NULL; 3729 spin_lock_irqsave(&zone->lock, flags); 3730 /* 3731 * order-0 request can reach here when the pcplist is skipped 3732 * due to non-CMA allocation context. HIGHATOMIC area is 3733 * reserved for high-order atomic allocation, so order-0 3734 * request should skip it. 3735 */ 3736 if (order > 0 && alloc_flags & ALLOC_HARDER) { 3737 page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC); 3738 if (page) 3739 trace_mm_page_alloc_zone_locked(page, order, migratetype); 3740 } 3741 if (!page) { 3742 page = __rmqueue(zone, order, migratetype, alloc_flags); 3743 if (!page) 3744 goto failed; 3745 } 3746 __mod_zone_freepage_state(zone, -(1 << order), 3747 get_pcppage_migratetype(page)); 3748 spin_unlock_irqrestore(&zone->lock, flags); 3749 } while (check_new_pages(page, order)); 3750 3751 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order); 3752 zone_statistics(preferred_zone, zone, 1); 3753 3754 out: 3755 /* Separate test+clear to avoid unnecessary atomics */ 3756 if (test_bit(ZONE_BOOSTED_WATERMARK, &zone->flags)) { 3757 clear_bit(ZONE_BOOSTED_WATERMARK, &zone->flags); 3758 wakeup_kswapd(zone, 0, 0, zone_idx(zone)); 3759 } 3760 3761 VM_BUG_ON_PAGE(page && bad_range(zone, page), page); 3762 return page; 3763 3764 failed: 3765 spin_unlock_irqrestore(&zone->lock, flags); 3766 return NULL; 3767 } 3768 3769 #ifdef CONFIG_FAIL_PAGE_ALLOC 3770 3771 static struct { 3772 struct fault_attr attr; 3773 3774 bool ignore_gfp_highmem; 3775 bool ignore_gfp_reclaim; 3776 u32 min_order; 3777 } fail_page_alloc = { 3778 .attr = FAULT_ATTR_INITIALIZER, 3779 .ignore_gfp_reclaim = true, 3780 .ignore_gfp_highmem = true, 3781 .min_order = 1, 3782 }; 3783 3784 static int __init setup_fail_page_alloc(char *str) 3785 { 3786 return setup_fault_attr(&fail_page_alloc.attr, str); 3787 } 3788 __setup("fail_page_alloc=", setup_fail_page_alloc); 3789 3790 static bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) 3791 { 3792 if (order < fail_page_alloc.min_order) 3793 return false; 3794 if (gfp_mask & __GFP_NOFAIL) 3795 return false; 3796 if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM)) 3797 return false; 3798 if (fail_page_alloc.ignore_gfp_reclaim && 3799 (gfp_mask & __GFP_DIRECT_RECLAIM)) 3800 return false; 3801 3802 return should_fail(&fail_page_alloc.attr, 1 << order); 3803 } 3804 3805 #ifdef CONFIG_FAULT_INJECTION_DEBUG_FS 3806 3807 static int __init fail_page_alloc_debugfs(void) 3808 { 3809 umode_t mode = S_IFREG | 0600; 3810 struct dentry *dir; 3811 3812 dir = fault_create_debugfs_attr("fail_page_alloc", NULL, 3813 &fail_page_alloc.attr); 3814 3815 debugfs_create_bool("ignore-gfp-wait", mode, dir, 3816 &fail_page_alloc.ignore_gfp_reclaim); 3817 debugfs_create_bool("ignore-gfp-highmem", mode, dir, 3818 &fail_page_alloc.ignore_gfp_highmem); 3819 debugfs_create_u32("min-order", mode, dir, &fail_page_alloc.min_order); 3820 3821 return 0; 3822 } 3823 3824 late_initcall(fail_page_alloc_debugfs); 3825 3826 #endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */ 3827 3828 #else /* CONFIG_FAIL_PAGE_ALLOC */ 3829 3830 static inline bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) 3831 { 3832 return false; 3833 } 3834 3835 #endif /* CONFIG_FAIL_PAGE_ALLOC */ 3836 3837 noinline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) 3838 { 3839 return __should_fail_alloc_page(gfp_mask, order); 3840 } 3841 ALLOW_ERROR_INJECTION(should_fail_alloc_page, TRUE); 3842 3843 static inline long __zone_watermark_unusable_free(struct zone *z, 3844 unsigned int order, unsigned int alloc_flags) 3845 { 3846 const bool alloc_harder = (alloc_flags & (ALLOC_HARDER|ALLOC_OOM)); 3847 long unusable_free = (1 << order) - 1; 3848 3849 /* 3850 * If the caller does not have rights to ALLOC_HARDER then subtract 3851 * the high-atomic reserves. This will over-estimate the size of the 3852 * atomic reserve but it avoids a search. 3853 */ 3854 if (likely(!alloc_harder)) 3855 unusable_free += z->nr_reserved_highatomic; 3856 3857 #ifdef CONFIG_CMA 3858 /* If allocation can't use CMA areas don't use free CMA pages */ 3859 if (!(alloc_flags & ALLOC_CMA)) 3860 unusable_free += zone_page_state(z, NR_FREE_CMA_PAGES); 3861 #endif 3862 3863 return unusable_free; 3864 } 3865 3866 /* 3867 * Return true if free base pages are above 'mark'. For high-order checks it 3868 * will return true of the order-0 watermark is reached and there is at least 3869 * one free page of a suitable size. Checking now avoids taking the zone lock 3870 * to check in the allocation paths if no pages are free. 3871 */ 3872 bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, 3873 int highest_zoneidx, unsigned int alloc_flags, 3874 long free_pages) 3875 { 3876 long min = mark; 3877 int o; 3878 const bool alloc_harder = (alloc_flags & (ALLOC_HARDER|ALLOC_OOM)); 3879 3880 /* free_pages may go negative - that's OK */ 3881 free_pages -= __zone_watermark_unusable_free(z, order, alloc_flags); 3882 3883 if (alloc_flags & ALLOC_HIGH) 3884 min -= min / 2; 3885 3886 if (unlikely(alloc_harder)) { 3887 /* 3888 * OOM victims can try even harder than normal ALLOC_HARDER 3889 * users on the grounds that it's definitely going to be in 3890 * the exit path shortly and free memory. Any allocation it 3891 * makes during the free path will be small and short-lived. 3892 */ 3893 if (alloc_flags & ALLOC_OOM) 3894 min -= min / 2; 3895 else 3896 min -= min / 4; 3897 } 3898 3899 /* 3900 * Check watermarks for an order-0 allocation request. If these 3901 * are not met, then a high-order request also cannot go ahead 3902 * even if a suitable page happened to be free. 3903 */ 3904 if (free_pages <= min + z->lowmem_reserve[highest_zoneidx]) 3905 return false; 3906 3907 /* If this is an order-0 request then the watermark is fine */ 3908 if (!order) 3909 return true; 3910 3911 /* For a high-order request, check at least one suitable page is free */ 3912 for (o = order; o < MAX_ORDER; o++) { 3913 struct free_area *area = &z->free_area[o]; 3914 int mt; 3915 3916 if (!area->nr_free) 3917 continue; 3918 3919 for (mt = 0; mt < MIGRATE_PCPTYPES; mt++) { 3920 if (!free_area_empty(area, mt)) 3921 return true; 3922 } 3923 3924 #ifdef CONFIG_CMA 3925 if ((alloc_flags & ALLOC_CMA) && 3926 !free_area_empty(area, MIGRATE_CMA)) { 3927 return true; 3928 } 3929 #endif 3930 if (alloc_harder && !free_area_empty(area, MIGRATE_HIGHATOMIC)) 3931 return true; 3932 } 3933 return false; 3934 } 3935 3936 bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, 3937 int highest_zoneidx, unsigned int alloc_flags) 3938 { 3939 return __zone_watermark_ok(z, order, mark, highest_zoneidx, alloc_flags, 3940 zone_page_state(z, NR_FREE_PAGES)); 3941 } 3942 3943 static inline bool zone_watermark_fast(struct zone *z, unsigned int order, 3944 unsigned long mark, int highest_zoneidx, 3945 unsigned int alloc_flags, gfp_t gfp_mask) 3946 { 3947 long free_pages; 3948 3949 free_pages = zone_page_state(z, NR_FREE_PAGES); 3950 3951 /* 3952 * Fast check for order-0 only. If this fails then the reserves 3953 * need to be calculated. 3954 */ 3955 if (!order) { 3956 long fast_free; 3957 3958 fast_free = free_pages; 3959 fast_free -= __zone_watermark_unusable_free(z, 0, alloc_flags); 3960 if (fast_free > mark + z->lowmem_reserve[highest_zoneidx]) 3961 return true; 3962 } 3963 3964 if (__zone_watermark_ok(z, order, mark, highest_zoneidx, alloc_flags, 3965 free_pages)) 3966 return true; 3967 /* 3968 * Ignore watermark boosting for GFP_ATOMIC order-0 allocations 3969 * when checking the min watermark. The min watermark is the 3970 * point where boosting is ignored so that kswapd is woken up 3971 * when below the low watermark. 3972 */ 3973 if (unlikely(!order && (gfp_mask & __GFP_ATOMIC) && z->watermark_boost 3974 && ((alloc_flags & ALLOC_WMARK_MASK) == WMARK_MIN))) { 3975 mark = z->_watermark[WMARK_MIN]; 3976 return __zone_watermark_ok(z, order, mark, highest_zoneidx, 3977 alloc_flags, free_pages); 3978 } 3979 3980 return false; 3981 } 3982 3983 bool zone_watermark_ok_safe(struct zone *z, unsigned int order, 3984 unsigned long mark, int highest_zoneidx) 3985 { 3986 long free_pages = zone_page_state(z, NR_FREE_PAGES); 3987 3988 if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark) 3989 free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES); 3990 3991 return __zone_watermark_ok(z, order, mark, highest_zoneidx, 0, 3992 free_pages); 3993 } 3994 3995 #ifdef CONFIG_NUMA 3996 int __read_mostly node_reclaim_distance = RECLAIM_DISTANCE; 3997 3998 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) 3999 { 4000 return node_distance(zone_to_nid(local_zone), zone_to_nid(zone)) <= 4001 node_reclaim_distance; 4002 } 4003 #else /* CONFIG_NUMA */ 4004 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) 4005 { 4006 return true; 4007 } 4008 #endif /* CONFIG_NUMA */ 4009 4010 /* 4011 * The restriction on ZONE_DMA32 as being a suitable zone to use to avoid 4012 * fragmentation is subtle. If the preferred zone was HIGHMEM then 4013 * premature use of a lower zone may cause lowmem pressure problems that 4014 * are worse than fragmentation. If the next zone is ZONE_DMA then it is 4015 * probably too small. It only makes sense to spread allocations to avoid 4016 * fragmentation between the Normal and DMA32 zones. 4017 */ 4018 static inline unsigned int 4019 alloc_flags_nofragment(struct zone *zone, gfp_t gfp_mask) 4020 { 4021 unsigned int alloc_flags; 4022 4023 /* 4024 * __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD 4025 * to save a branch. 4026 */ 4027 alloc_flags = (__force int) (gfp_mask & __GFP_KSWAPD_RECLAIM); 4028 4029 #ifdef CONFIG_ZONE_DMA32 4030 if (!zone) 4031 return alloc_flags; 4032 4033 if (zone_idx(zone) != ZONE_NORMAL) 4034 return alloc_flags; 4035 4036 /* 4037 * If ZONE_DMA32 exists, assume it is the one after ZONE_NORMAL and 4038 * the pointer is within zone->zone_pgdat->node_zones[]. Also assume 4039 * on UMA that if Normal is populated then so is DMA32. 4040 */ 4041 BUILD_BUG_ON(ZONE_NORMAL - ZONE_DMA32 != 1); 4042 if (nr_online_nodes > 1 && !populated_zone(--zone)) 4043 return alloc_flags; 4044 4045 alloc_flags |= ALLOC_NOFRAGMENT; 4046 #endif /* CONFIG_ZONE_DMA32 */ 4047 return alloc_flags; 4048 } 4049 4050 /* Must be called after current_gfp_context() which can change gfp_mask */ 4051 static inline unsigned int gfp_to_alloc_flags_cma(gfp_t gfp_mask, 4052 unsigned int alloc_flags) 4053 { 4054 #ifdef CONFIG_CMA 4055 if (gfp_migratetype(gfp_mask) == MIGRATE_MOVABLE) 4056 alloc_flags |= ALLOC_CMA; 4057 #endif 4058 return alloc_flags; 4059 } 4060 4061 /* 4062 * get_page_from_freelist goes through the zonelist trying to allocate 4063 * a page. 4064 */ 4065 static struct page * 4066 get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags, 4067 const struct alloc_context *ac) 4068 { 4069 struct zoneref *z; 4070 struct zone *zone; 4071 struct pglist_data *last_pgdat_dirty_limit = NULL; 4072 bool no_fallback; 4073 4074 retry: 4075 /* 4076 * Scan zonelist, looking for a zone with enough free. 4077 * See also __cpuset_node_allowed() comment in kernel/cpuset.c. 4078 */ 4079 no_fallback = alloc_flags & ALLOC_NOFRAGMENT; 4080 z = ac->preferred_zoneref; 4081 for_next_zone_zonelist_nodemask(zone, z, ac->highest_zoneidx, 4082 ac->nodemask) { 4083 struct page *page; 4084 unsigned long mark; 4085 4086 if (cpusets_enabled() && 4087 (alloc_flags & ALLOC_CPUSET) && 4088 !__cpuset_zone_allowed(zone, gfp_mask)) 4089 continue; 4090 /* 4091 * When allocating a page cache page for writing, we 4092 * want to get it from a node that is within its dirty 4093 * limit, such that no single node holds more than its 4094 * proportional share of globally allowed dirty pages. 4095 * The dirty limits take into account the node's 4096 * lowmem reserves and high watermark so that kswapd 4097 * should be able to balance it without having to 4098 * write pages from its LRU list. 4099 * 4100 * XXX: For now, allow allocations to potentially 4101 * exceed the per-node dirty limit in the slowpath 4102 * (spread_dirty_pages unset) before going into reclaim, 4103 * which is important when on a NUMA setup the allowed 4104 * nodes are together not big enough to reach the 4105 * global limit. The proper fix for these situations 4106 * will require awareness of nodes in the 4107 * dirty-throttling and the flusher threads. 4108 */ 4109 if (ac->spread_dirty_pages) { 4110 if (last_pgdat_dirty_limit == zone->zone_pgdat) 4111 continue; 4112 4113 if (!node_dirty_ok(zone->zone_pgdat)) { 4114 last_pgdat_dirty_limit = zone->zone_pgdat; 4115 continue; 4116 } 4117 } 4118 4119 if (no_fallback && nr_online_nodes > 1 && 4120 zone != ac->preferred_zoneref->zone) { 4121 int local_nid; 4122 4123 /* 4124 * If moving to a remote node, retry but allow 4125 * fragmenting fallbacks. Locality is more important 4126 * than fragmentation avoidance. 4127 */ 4128 local_nid = zone_to_nid(ac->preferred_zoneref->zone); 4129 if (zone_to_nid(zone) != local_nid) { 4130 alloc_flags &= ~ALLOC_NOFRAGMENT; 4131 goto retry; 4132 } 4133 } 4134 4135 mark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK); 4136 if (!zone_watermark_fast(zone, order, mark, 4137 ac->highest_zoneidx, alloc_flags, 4138 gfp_mask)) { 4139 int ret; 4140 4141 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 4142 /* 4143 * Watermark failed for this zone, but see if we can 4144 * grow this zone if it contains deferred pages. 4145 */ 4146 if (static_branch_unlikely(&deferred_pages)) { 4147 if (_deferred_grow_zone(zone, order)) 4148 goto try_this_zone; 4149 } 4150 #endif 4151 /* Checked here to keep the fast path fast */ 4152 BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK); 4153 if (alloc_flags & ALLOC_NO_WATERMARKS) 4154 goto try_this_zone; 4155 4156 if (!node_reclaim_enabled() || 4157 !zone_allows_reclaim(ac->preferred_zoneref->zone, zone)) 4158 continue; 4159 4160 ret = node_reclaim(zone->zone_pgdat, gfp_mask, order); 4161 switch (ret) { 4162 case NODE_RECLAIM_NOSCAN: 4163 /* did not scan */ 4164 continue; 4165 case NODE_RECLAIM_FULL: 4166 /* scanned but unreclaimable */ 4167 continue; 4168 default: 4169 /* did we reclaim enough */ 4170 if (zone_watermark_ok(zone, order, mark, 4171 ac->highest_zoneidx, alloc_flags)) 4172 goto try_this_zone; 4173 4174 continue; 4175 } 4176 } 4177 4178 try_this_zone: 4179 page = rmqueue(ac->preferred_zoneref->zone, zone, order, 4180 gfp_mask, alloc_flags, ac->migratetype); 4181 if (page) { 4182 prep_new_page(page, order, gfp_mask, alloc_flags); 4183 4184 /* 4185 * If this is a high-order atomic allocation then check 4186 * if the pageblock should be reserved for the future 4187 */ 4188 if (unlikely(order && (alloc_flags & ALLOC_HARDER))) 4189 reserve_highatomic_pageblock(page, zone, order); 4190 4191 return page; 4192 } else { 4193 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 4194 /* Try again if zone has deferred pages */ 4195 if (static_branch_unlikely(&deferred_pages)) { 4196 if (_deferred_grow_zone(zone, order)) 4197 goto try_this_zone; 4198 } 4199 #endif 4200 } 4201 } 4202 4203 /* 4204 * It's possible on a UMA machine to get through all zones that are 4205 * fragmented. If avoiding fragmentation, reset and try again. 4206 */ 4207 if (no_fallback) { 4208 alloc_flags &= ~ALLOC_NOFRAGMENT; 4209 goto retry; 4210 } 4211 4212 return NULL; 4213 } 4214 4215 static void warn_alloc_show_mem(gfp_t gfp_mask, nodemask_t *nodemask) 4216 { 4217 unsigned int filter = SHOW_MEM_FILTER_NODES; 4218 4219 /* 4220 * This documents exceptions given to allocations in certain 4221 * contexts that are allowed to allocate outside current's set 4222 * of allowed nodes. 4223 */ 4224 if (!(gfp_mask & __GFP_NOMEMALLOC)) 4225 if (tsk_is_oom_victim(current) || 4226 (current->flags & (PF_MEMALLOC | PF_EXITING))) 4227 filter &= ~SHOW_MEM_FILTER_NODES; 4228 if (!in_task() || !(gfp_mask & __GFP_DIRECT_RECLAIM)) 4229 filter &= ~SHOW_MEM_FILTER_NODES; 4230 4231 show_mem(filter, nodemask); 4232 } 4233 4234 void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...) 4235 { 4236 struct va_format vaf; 4237 va_list args; 4238 static DEFINE_RATELIMIT_STATE(nopage_rs, 10*HZ, 1); 4239 4240 if ((gfp_mask & __GFP_NOWARN) || 4241 !__ratelimit(&nopage_rs) || 4242 ((gfp_mask & __GFP_DMA) && !has_managed_dma())) 4243 return; 4244 4245 va_start(args, fmt); 4246 vaf.fmt = fmt; 4247 vaf.va = &args; 4248 pr_warn("%s: %pV, mode:%#x(%pGg), nodemask=%*pbl", 4249 current->comm, &vaf, gfp_mask, &gfp_mask, 4250 nodemask_pr_args(nodemask)); 4251 va_end(args); 4252 4253 cpuset_print_current_mems_allowed(); 4254 pr_cont("\n"); 4255 dump_stack(); 4256 warn_alloc_show_mem(gfp_mask, nodemask); 4257 } 4258 4259 static inline struct page * 4260 __alloc_pages_cpuset_fallback(gfp_t gfp_mask, unsigned int order, 4261 unsigned int alloc_flags, 4262 const struct alloc_context *ac) 4263 { 4264 struct page *page; 4265 4266 page = get_page_from_freelist(gfp_mask, order, 4267 alloc_flags|ALLOC_CPUSET, ac); 4268 /* 4269 * fallback to ignore cpuset restriction if our nodes 4270 * are depleted 4271 */ 4272 if (!page) 4273 page = get_page_from_freelist(gfp_mask, order, 4274 alloc_flags, ac); 4275 4276 return page; 4277 } 4278 4279 static inline struct page * 4280 __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order, 4281 const struct alloc_context *ac, unsigned long *did_some_progress) 4282 { 4283 struct oom_control oc = { 4284 .zonelist = ac->zonelist, 4285 .nodemask = ac->nodemask, 4286 .memcg = NULL, 4287 .gfp_mask = gfp_mask, 4288 .order = order, 4289 }; 4290 struct page *page; 4291 4292 *did_some_progress = 0; 4293 4294 /* 4295 * Acquire the oom lock. If that fails, somebody else is 4296 * making progress for us. 4297 */ 4298 if (!mutex_trylock(&oom_lock)) { 4299 *did_some_progress = 1; 4300 schedule_timeout_uninterruptible(1); 4301 return NULL; 4302 } 4303 4304 /* 4305 * Go through the zonelist yet one more time, keep very high watermark 4306 * here, this is only to catch a parallel oom killing, we must fail if 4307 * we're still under heavy pressure. But make sure that this reclaim 4308 * attempt shall not depend on __GFP_DIRECT_RECLAIM && !__GFP_NORETRY 4309 * allocation which will never fail due to oom_lock already held. 4310 */ 4311 page = get_page_from_freelist((gfp_mask | __GFP_HARDWALL) & 4312 ~__GFP_DIRECT_RECLAIM, order, 4313 ALLOC_WMARK_HIGH|ALLOC_CPUSET, ac); 4314 if (page) 4315 goto out; 4316 4317 /* Coredumps can quickly deplete all memory reserves */ 4318 if (current->flags & PF_DUMPCORE) 4319 goto out; 4320 /* The OOM killer will not help higher order allocs */ 4321 if (order > PAGE_ALLOC_COSTLY_ORDER) 4322 goto out; 4323 /* 4324 * We have already exhausted all our reclaim opportunities without any 4325 * success so it is time to admit defeat. We will skip the OOM killer 4326 * because it is very likely that the caller has a more reasonable 4327 * fallback than shooting a random task. 4328 * 4329 * The OOM killer may not free memory on a specific node. 4330 */ 4331 if (gfp_mask & (__GFP_RETRY_MAYFAIL | __GFP_THISNODE)) 4332 goto out; 4333 /* The OOM killer does not needlessly kill tasks for lowmem */ 4334 if (ac->highest_zoneidx < ZONE_NORMAL) 4335 goto out; 4336 if (pm_suspended_storage()) 4337 goto out; 4338 /* 4339 * XXX: GFP_NOFS allocations should rather fail than rely on 4340 * other request to make a forward progress. 4341 * We are in an unfortunate situation where out_of_memory cannot 4342 * do much for this context but let's try it to at least get 4343 * access to memory reserved if the current task is killed (see 4344 * out_of_memory). Once filesystems are ready to handle allocation 4345 * failures more gracefully we should just bail out here. 4346 */ 4347 4348 /* Exhausted what can be done so it's blame time */ 4349 if (out_of_memory(&oc) || WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL)) { 4350 *did_some_progress = 1; 4351 4352 /* 4353 * Help non-failing allocations by giving them access to memory 4354 * reserves 4355 */ 4356 if (gfp_mask & __GFP_NOFAIL) 4357 page = __alloc_pages_cpuset_fallback(gfp_mask, order, 4358 ALLOC_NO_WATERMARKS, ac); 4359 } 4360 out: 4361 mutex_unlock(&oom_lock); 4362 return page; 4363 } 4364 4365 /* 4366 * Maximum number of compaction retries with a progress before OOM 4367 * killer is consider as the only way to move forward. 4368 */ 4369 #define MAX_COMPACT_RETRIES 16 4370 4371 #ifdef CONFIG_COMPACTION 4372 /* Try memory compaction for high-order allocations before reclaim */ 4373 static struct page * 4374 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, 4375 unsigned int alloc_flags, const struct alloc_context *ac, 4376 enum compact_priority prio, enum compact_result *compact_result) 4377 { 4378 struct page *page = NULL; 4379 unsigned long pflags; 4380 unsigned int noreclaim_flag; 4381 4382 if (!order) 4383 return NULL; 4384 4385 psi_memstall_enter(&pflags); 4386 delayacct_compact_start(); 4387 noreclaim_flag = memalloc_noreclaim_save(); 4388 4389 *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac, 4390 prio, &page); 4391 4392 memalloc_noreclaim_restore(noreclaim_flag); 4393 psi_memstall_leave(&pflags); 4394 delayacct_compact_end(); 4395 4396 if (*compact_result == COMPACT_SKIPPED) 4397 return NULL; 4398 /* 4399 * At least in one zone compaction wasn't deferred or skipped, so let's 4400 * count a compaction stall 4401 */ 4402 count_vm_event(COMPACTSTALL); 4403 4404 /* Prep a captured page if available */ 4405 if (page) 4406 prep_new_page(page, order, gfp_mask, alloc_flags); 4407 4408 /* Try get a page from the freelist if available */ 4409 if (!page) 4410 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 4411 4412 if (page) { 4413 struct zone *zone = page_zone(page); 4414 4415 zone->compact_blockskip_flush = false; 4416 compaction_defer_reset(zone, order, true); 4417 count_vm_event(COMPACTSUCCESS); 4418 return page; 4419 } 4420 4421 /* 4422 * It's bad if compaction run occurs and fails. The most likely reason 4423 * is that pages exist, but not enough to satisfy watermarks. 4424 */ 4425 count_vm_event(COMPACTFAIL); 4426 4427 cond_resched(); 4428 4429 return NULL; 4430 } 4431 4432 static inline bool 4433 should_compact_retry(struct alloc_context *ac, int order, int alloc_flags, 4434 enum compact_result compact_result, 4435 enum compact_priority *compact_priority, 4436 int *compaction_retries) 4437 { 4438 int max_retries = MAX_COMPACT_RETRIES; 4439 int min_priority; 4440 bool ret = false; 4441 int retries = *compaction_retries; 4442 enum compact_priority priority = *compact_priority; 4443 4444 if (!order) 4445 return false; 4446 4447 if (fatal_signal_pending(current)) 4448 return false; 4449 4450 if (compaction_made_progress(compact_result)) 4451 (*compaction_retries)++; 4452 4453 /* 4454 * compaction considers all the zone as desperately out of memory 4455 * so it doesn't really make much sense to retry except when the 4456 * failure could be caused by insufficient priority 4457 */ 4458 if (compaction_failed(compact_result)) 4459 goto check_priority; 4460 4461 /* 4462 * compaction was skipped because there are not enough order-0 pages 4463 * to work with, so we retry only if it looks like reclaim can help. 4464 */ 4465 if (compaction_needs_reclaim(compact_result)) { 4466 ret = compaction_zonelist_suitable(ac, order, alloc_flags); 4467 goto out; 4468 } 4469 4470 /* 4471 * make sure the compaction wasn't deferred or didn't bail out early 4472 * due to locks contention before we declare that we should give up. 4473 * But the next retry should use a higher priority if allowed, so 4474 * we don't just keep bailing out endlessly. 4475 */ 4476 if (compaction_withdrawn(compact_result)) { 4477 goto check_priority; 4478 } 4479 4480 /* 4481 * !costly requests are much more important than __GFP_RETRY_MAYFAIL 4482 * costly ones because they are de facto nofail and invoke OOM 4483 * killer to move on while costly can fail and users are ready 4484 * to cope with that. 1/4 retries is rather arbitrary but we 4485 * would need much more detailed feedback from compaction to 4486 * make a better decision. 4487 */ 4488 if (order > PAGE_ALLOC_COSTLY_ORDER) 4489 max_retries /= 4; 4490 if (*compaction_retries <= max_retries) { 4491 ret = true; 4492 goto out; 4493 } 4494 4495 /* 4496 * Make sure there are attempts at the highest priority if we exhausted 4497 * all retries or failed at the lower priorities. 4498 */ 4499 check_priority: 4500 min_priority = (order > PAGE_ALLOC_COSTLY_ORDER) ? 4501 MIN_COMPACT_COSTLY_PRIORITY : MIN_COMPACT_PRIORITY; 4502 4503 if (*compact_priority > min_priority) { 4504 (*compact_priority)--; 4505 *compaction_retries = 0; 4506 ret = true; 4507 } 4508 out: 4509 trace_compact_retry(order, priority, compact_result, retries, max_retries, ret); 4510 return ret; 4511 } 4512 #else 4513 static inline struct page * 4514 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, 4515 unsigned int alloc_flags, const struct alloc_context *ac, 4516 enum compact_priority prio, enum compact_result *compact_result) 4517 { 4518 *compact_result = COMPACT_SKIPPED; 4519 return NULL; 4520 } 4521 4522 static inline bool 4523 should_compact_retry(struct alloc_context *ac, unsigned int order, int alloc_flags, 4524 enum compact_result compact_result, 4525 enum compact_priority *compact_priority, 4526 int *compaction_retries) 4527 { 4528 struct zone *zone; 4529 struct zoneref *z; 4530 4531 if (!order || order > PAGE_ALLOC_COSTLY_ORDER) 4532 return false; 4533 4534 /* 4535 * There are setups with compaction disabled which would prefer to loop 4536 * inside the allocator rather than hit the oom killer prematurely. 4537 * Let's give them a good hope and keep retrying while the order-0 4538 * watermarks are OK. 4539 */ 4540 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, 4541 ac->highest_zoneidx, ac->nodemask) { 4542 if (zone_watermark_ok(zone, 0, min_wmark_pages(zone), 4543 ac->highest_zoneidx, alloc_flags)) 4544 return true; 4545 } 4546 return false; 4547 } 4548 #endif /* CONFIG_COMPACTION */ 4549 4550 #ifdef CONFIG_LOCKDEP 4551 static struct lockdep_map __fs_reclaim_map = 4552 STATIC_LOCKDEP_MAP_INIT("fs_reclaim", &__fs_reclaim_map); 4553 4554 static bool __need_reclaim(gfp_t gfp_mask) 4555 { 4556 /* no reclaim without waiting on it */ 4557 if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) 4558 return false; 4559 4560 /* this guy won't enter reclaim */ 4561 if (current->flags & PF_MEMALLOC) 4562 return false; 4563 4564 if (gfp_mask & __GFP_NOLOCKDEP) 4565 return false; 4566 4567 return true; 4568 } 4569 4570 void __fs_reclaim_acquire(unsigned long ip) 4571 { 4572 lock_acquire_exclusive(&__fs_reclaim_map, 0, 0, NULL, ip); 4573 } 4574 4575 void __fs_reclaim_release(unsigned long ip) 4576 { 4577 lock_release(&__fs_reclaim_map, ip); 4578 } 4579 4580 void fs_reclaim_acquire(gfp_t gfp_mask) 4581 { 4582 gfp_mask = current_gfp_context(gfp_mask); 4583 4584 if (__need_reclaim(gfp_mask)) { 4585 if (gfp_mask & __GFP_FS) 4586 __fs_reclaim_acquire(_RET_IP_); 4587 4588 #ifdef CONFIG_MMU_NOTIFIER 4589 lock_map_acquire(&__mmu_notifier_invalidate_range_start_map); 4590 lock_map_release(&__mmu_notifier_invalidate_range_start_map); 4591 #endif 4592 4593 } 4594 } 4595 EXPORT_SYMBOL_GPL(fs_reclaim_acquire); 4596 4597 void fs_reclaim_release(gfp_t gfp_mask) 4598 { 4599 gfp_mask = current_gfp_context(gfp_mask); 4600 4601 if (__need_reclaim(gfp_mask)) { 4602 if (gfp_mask & __GFP_FS) 4603 __fs_reclaim_release(_RET_IP_); 4604 } 4605 } 4606 EXPORT_SYMBOL_GPL(fs_reclaim_release); 4607 #endif 4608 4609 /* Perform direct synchronous page reclaim */ 4610 static unsigned long 4611 __perform_reclaim(gfp_t gfp_mask, unsigned int order, 4612 const struct alloc_context *ac) 4613 { 4614 unsigned int noreclaim_flag; 4615 unsigned long progress; 4616 4617 cond_resched(); 4618 4619 /* We now go into synchronous reclaim */ 4620 cpuset_memory_pressure_bump(); 4621 fs_reclaim_acquire(gfp_mask); 4622 noreclaim_flag = memalloc_noreclaim_save(); 4623 4624 progress = try_to_free_pages(ac->zonelist, order, gfp_mask, 4625 ac->nodemask); 4626 4627 memalloc_noreclaim_restore(noreclaim_flag); 4628 fs_reclaim_release(gfp_mask); 4629 4630 cond_resched(); 4631 4632 return progress; 4633 } 4634 4635 /* The really slow allocator path where we enter direct reclaim */ 4636 static inline struct page * 4637 __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order, 4638 unsigned int alloc_flags, const struct alloc_context *ac, 4639 unsigned long *did_some_progress) 4640 { 4641 struct page *page = NULL; 4642 unsigned long pflags; 4643 bool drained = false; 4644 4645 psi_memstall_enter(&pflags); 4646 *did_some_progress = __perform_reclaim(gfp_mask, order, ac); 4647 if (unlikely(!(*did_some_progress))) 4648 goto out; 4649 4650 retry: 4651 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 4652 4653 /* 4654 * If an allocation failed after direct reclaim, it could be because 4655 * pages are pinned on the per-cpu lists or in high alloc reserves. 4656 * Shrink them and try again 4657 */ 4658 if (!page && !drained) { 4659 unreserve_highatomic_pageblock(ac, false); 4660 drain_all_pages(NULL); 4661 drained = true; 4662 goto retry; 4663 } 4664 out: 4665 psi_memstall_leave(&pflags); 4666 4667 return page; 4668 } 4669 4670 static void wake_all_kswapds(unsigned int order, gfp_t gfp_mask, 4671 const struct alloc_context *ac) 4672 { 4673 struct zoneref *z; 4674 struct zone *zone; 4675 pg_data_t *last_pgdat = NULL; 4676 enum zone_type highest_zoneidx = ac->highest_zoneidx; 4677 4678 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, highest_zoneidx, 4679 ac->nodemask) { 4680 if (last_pgdat != zone->zone_pgdat) 4681 wakeup_kswapd(zone, gfp_mask, order, highest_zoneidx); 4682 last_pgdat = zone->zone_pgdat; 4683 } 4684 } 4685 4686 static inline unsigned int 4687 gfp_to_alloc_flags(gfp_t gfp_mask) 4688 { 4689 unsigned int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET; 4690 4691 /* 4692 * __GFP_HIGH is assumed to be the same as ALLOC_HIGH 4693 * and __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD 4694 * to save two branches. 4695 */ 4696 BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_HIGH); 4697 BUILD_BUG_ON(__GFP_KSWAPD_RECLAIM != (__force gfp_t) ALLOC_KSWAPD); 4698 4699 /* 4700 * The caller may dip into page reserves a bit more if the caller 4701 * cannot run direct reclaim, or if the caller has realtime scheduling 4702 * policy or is asking for __GFP_HIGH memory. GFP_ATOMIC requests will 4703 * set both ALLOC_HARDER (__GFP_ATOMIC) and ALLOC_HIGH (__GFP_HIGH). 4704 */ 4705 alloc_flags |= (__force int) 4706 (gfp_mask & (__GFP_HIGH | __GFP_KSWAPD_RECLAIM)); 4707 4708 if (gfp_mask & __GFP_ATOMIC) { 4709 /* 4710 * Not worth trying to allocate harder for __GFP_NOMEMALLOC even 4711 * if it can't schedule. 4712 */ 4713 if (!(gfp_mask & __GFP_NOMEMALLOC)) 4714 alloc_flags |= ALLOC_HARDER; 4715 /* 4716 * Ignore cpuset mems for GFP_ATOMIC rather than fail, see the 4717 * comment for __cpuset_node_allowed(). 4718 */ 4719 alloc_flags &= ~ALLOC_CPUSET; 4720 } else if (unlikely(rt_task(current)) && in_task()) 4721 alloc_flags |= ALLOC_HARDER; 4722 4723 alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, alloc_flags); 4724 4725 return alloc_flags; 4726 } 4727 4728 static bool oom_reserves_allowed(struct task_struct *tsk) 4729 { 4730 if (!tsk_is_oom_victim(tsk)) 4731 return false; 4732 4733 /* 4734 * !MMU doesn't have oom reaper so give access to memory reserves 4735 * only to the thread with TIF_MEMDIE set 4736 */ 4737 if (!IS_ENABLED(CONFIG_MMU) && !test_thread_flag(TIF_MEMDIE)) 4738 return false; 4739 4740 return true; 4741 } 4742 4743 /* 4744 * Distinguish requests which really need access to full memory 4745 * reserves from oom victims which can live with a portion of it 4746 */ 4747 static inline int __gfp_pfmemalloc_flags(gfp_t gfp_mask) 4748 { 4749 if (unlikely(gfp_mask & __GFP_NOMEMALLOC)) 4750 return 0; 4751 if (gfp_mask & __GFP_MEMALLOC) 4752 return ALLOC_NO_WATERMARKS; 4753 if (in_serving_softirq() && (current->flags & PF_MEMALLOC)) 4754 return ALLOC_NO_WATERMARKS; 4755 if (!in_interrupt()) { 4756 if (current->flags & PF_MEMALLOC) 4757 return ALLOC_NO_WATERMARKS; 4758 else if (oom_reserves_allowed(current)) 4759 return ALLOC_OOM; 4760 } 4761 4762 return 0; 4763 } 4764 4765 bool gfp_pfmemalloc_allowed(gfp_t gfp_mask) 4766 { 4767 return !!__gfp_pfmemalloc_flags(gfp_mask); 4768 } 4769 4770 /* 4771 * Checks whether it makes sense to retry the reclaim to make a forward progress 4772 * for the given allocation request. 4773 * 4774 * We give up when we either have tried MAX_RECLAIM_RETRIES in a row 4775 * without success, or when we couldn't even meet the watermark if we 4776 * reclaimed all remaining pages on the LRU lists. 4777 * 4778 * Returns true if a retry is viable or false to enter the oom path. 4779 */ 4780 static inline bool 4781 should_reclaim_retry(gfp_t gfp_mask, unsigned order, 4782 struct alloc_context *ac, int alloc_flags, 4783 bool did_some_progress, int *no_progress_loops) 4784 { 4785 struct zone *zone; 4786 struct zoneref *z; 4787 bool ret = false; 4788 4789 /* 4790 * Costly allocations might have made a progress but this doesn't mean 4791 * their order will become available due to high fragmentation so 4792 * always increment the no progress counter for them 4793 */ 4794 if (did_some_progress && order <= PAGE_ALLOC_COSTLY_ORDER) 4795 *no_progress_loops = 0; 4796 else 4797 (*no_progress_loops)++; 4798 4799 /* 4800 * Make sure we converge to OOM if we cannot make any progress 4801 * several times in the row. 4802 */ 4803 if (*no_progress_loops > MAX_RECLAIM_RETRIES) { 4804 /* Before OOM, exhaust highatomic_reserve */ 4805 return unreserve_highatomic_pageblock(ac, true); 4806 } 4807 4808 /* 4809 * Keep reclaiming pages while there is a chance this will lead 4810 * somewhere. If none of the target zones can satisfy our allocation 4811 * request even if all reclaimable pages are considered then we are 4812 * screwed and have to go OOM. 4813 */ 4814 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, 4815 ac->highest_zoneidx, ac->nodemask) { 4816 unsigned long available; 4817 unsigned long reclaimable; 4818 unsigned long min_wmark = min_wmark_pages(zone); 4819 bool wmark; 4820 4821 available = reclaimable = zone_reclaimable_pages(zone); 4822 available += zone_page_state_snapshot(zone, NR_FREE_PAGES); 4823 4824 /* 4825 * Would the allocation succeed if we reclaimed all 4826 * reclaimable pages? 4827 */ 4828 wmark = __zone_watermark_ok(zone, order, min_wmark, 4829 ac->highest_zoneidx, alloc_flags, available); 4830 trace_reclaim_retry_zone(z, order, reclaimable, 4831 available, min_wmark, *no_progress_loops, wmark); 4832 if (wmark) { 4833 ret = true; 4834 break; 4835 } 4836 } 4837 4838 /* 4839 * Memory allocation/reclaim might be called from a WQ context and the 4840 * current implementation of the WQ concurrency control doesn't 4841 * recognize that a particular WQ is congested if the worker thread is 4842 * looping without ever sleeping. Therefore we have to do a short sleep 4843 * here rather than calling cond_resched(). 4844 */ 4845 if (current->flags & PF_WQ_WORKER) 4846 schedule_timeout_uninterruptible(1); 4847 else 4848 cond_resched(); 4849 return ret; 4850 } 4851 4852 static inline bool 4853 check_retry_cpuset(int cpuset_mems_cookie, struct alloc_context *ac) 4854 { 4855 /* 4856 * It's possible that cpuset's mems_allowed and the nodemask from 4857 * mempolicy don't intersect. This should be normally dealt with by 4858 * policy_nodemask(), but it's possible to race with cpuset update in 4859 * such a way the check therein was true, and then it became false 4860 * before we got our cpuset_mems_cookie here. 4861 * This assumes that for all allocations, ac->nodemask can come only 4862 * from MPOL_BIND mempolicy (whose documented semantics is to be ignored 4863 * when it does not intersect with the cpuset restrictions) or the 4864 * caller can deal with a violated nodemask. 4865 */ 4866 if (cpusets_enabled() && ac->nodemask && 4867 !cpuset_nodemask_valid_mems_allowed(ac->nodemask)) { 4868 ac->nodemask = NULL; 4869 return true; 4870 } 4871 4872 /* 4873 * When updating a task's mems_allowed or mempolicy nodemask, it is 4874 * possible to race with parallel threads in such a way that our 4875 * allocation can fail while the mask is being updated. If we are about 4876 * to fail, check if the cpuset changed during allocation and if so, 4877 * retry. 4878 */ 4879 if (read_mems_allowed_retry(cpuset_mems_cookie)) 4880 return true; 4881 4882 return false; 4883 } 4884 4885 static inline struct page * 4886 __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, 4887 struct alloc_context *ac) 4888 { 4889 bool can_direct_reclaim = gfp_mask & __GFP_DIRECT_RECLAIM; 4890 const bool costly_order = order > PAGE_ALLOC_COSTLY_ORDER; 4891 struct page *page = NULL; 4892 unsigned int alloc_flags; 4893 unsigned long did_some_progress; 4894 enum compact_priority compact_priority; 4895 enum compact_result compact_result; 4896 int compaction_retries; 4897 int no_progress_loops; 4898 unsigned int cpuset_mems_cookie; 4899 int reserve_flags; 4900 4901 /* 4902 * We also sanity check to catch abuse of atomic reserves being used by 4903 * callers that are not in atomic context. 4904 */ 4905 if (WARN_ON_ONCE((gfp_mask & (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)) == 4906 (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM))) 4907 gfp_mask &= ~__GFP_ATOMIC; 4908 4909 retry_cpuset: 4910 compaction_retries = 0; 4911 no_progress_loops = 0; 4912 compact_priority = DEF_COMPACT_PRIORITY; 4913 cpuset_mems_cookie = read_mems_allowed_begin(); 4914 4915 /* 4916 * The fast path uses conservative alloc_flags to succeed only until 4917 * kswapd needs to be woken up, and to avoid the cost of setting up 4918 * alloc_flags precisely. So we do that now. 4919 */ 4920 alloc_flags = gfp_to_alloc_flags(gfp_mask); 4921 4922 /* 4923 * We need to recalculate the starting point for the zonelist iterator 4924 * because we might have used different nodemask in the fast path, or 4925 * there was a cpuset modification and we are retrying - otherwise we 4926 * could end up iterating over non-eligible zones endlessly. 4927 */ 4928 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist, 4929 ac->highest_zoneidx, ac->nodemask); 4930 if (!ac->preferred_zoneref->zone) 4931 goto nopage; 4932 4933 /* 4934 * Check for insane configurations where the cpuset doesn't contain 4935 * any suitable zone to satisfy the request - e.g. non-movable 4936 * GFP_HIGHUSER allocations from MOVABLE nodes only. 4937 */ 4938 if (cpusets_insane_config() && (gfp_mask & __GFP_HARDWALL)) { 4939 struct zoneref *z = first_zones_zonelist(ac->zonelist, 4940 ac->highest_zoneidx, 4941 &cpuset_current_mems_allowed); 4942 if (!z->zone) 4943 goto nopage; 4944 } 4945 4946 if (alloc_flags & ALLOC_KSWAPD) 4947 wake_all_kswapds(order, gfp_mask, ac); 4948 4949 /* 4950 * The adjusted alloc_flags might result in immediate success, so try 4951 * that first 4952 */ 4953 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 4954 if (page) 4955 goto got_pg; 4956 4957 /* 4958 * For costly allocations, try direct compaction first, as it's likely 4959 * that we have enough base pages and don't need to reclaim. For non- 4960 * movable high-order allocations, do that as well, as compaction will 4961 * try prevent permanent fragmentation by migrating from blocks of the 4962 * same migratetype. 4963 * Don't try this for allocations that are allowed to ignore 4964 * watermarks, as the ALLOC_NO_WATERMARKS attempt didn't yet happen. 4965 */ 4966 if (can_direct_reclaim && 4967 (costly_order || 4968 (order > 0 && ac->migratetype != MIGRATE_MOVABLE)) 4969 && !gfp_pfmemalloc_allowed(gfp_mask)) { 4970 page = __alloc_pages_direct_compact(gfp_mask, order, 4971 alloc_flags, ac, 4972 INIT_COMPACT_PRIORITY, 4973 &compact_result); 4974 if (page) 4975 goto got_pg; 4976 4977 /* 4978 * Checks for costly allocations with __GFP_NORETRY, which 4979 * includes some THP page fault allocations 4980 */ 4981 if (costly_order && (gfp_mask & __GFP_NORETRY)) { 4982 /* 4983 * If allocating entire pageblock(s) and compaction 4984 * failed because all zones are below low watermarks 4985 * or is prohibited because it recently failed at this 4986 * order, fail immediately unless the allocator has 4987 * requested compaction and reclaim retry. 4988 * 4989 * Reclaim is 4990 * - potentially very expensive because zones are far 4991 * below their low watermarks or this is part of very 4992 * bursty high order allocations, 4993 * - not guaranteed to help because isolate_freepages() 4994 * may not iterate over freed pages as part of its 4995 * linear scan, and 4996 * - unlikely to make entire pageblocks free on its 4997 * own. 4998 */ 4999 if (compact_result == COMPACT_SKIPPED || 5000 compact_result == COMPACT_DEFERRED) 5001 goto nopage; 5002 5003 /* 5004 * Looks like reclaim/compaction is worth trying, but 5005 * sync compaction could be very expensive, so keep 5006 * using async compaction. 5007 */ 5008 compact_priority = INIT_COMPACT_PRIORITY; 5009 } 5010 } 5011 5012 retry: 5013 /* Ensure kswapd doesn't accidentally go to sleep as long as we loop */ 5014 if (alloc_flags & ALLOC_KSWAPD) 5015 wake_all_kswapds(order, gfp_mask, ac); 5016 5017 reserve_flags = __gfp_pfmemalloc_flags(gfp_mask); 5018 if (reserve_flags) 5019 alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, reserve_flags); 5020 5021 /* 5022 * Reset the nodemask and zonelist iterators if memory policies can be 5023 * ignored. These allocations are high priority and system rather than 5024 * user oriented. 5025 */ 5026 if (!(alloc_flags & ALLOC_CPUSET) || reserve_flags) { 5027 ac->nodemask = NULL; 5028 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist, 5029 ac->highest_zoneidx, ac->nodemask); 5030 } 5031 5032 /* Attempt with potentially adjusted zonelist and alloc_flags */ 5033 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 5034 if (page) 5035 goto got_pg; 5036 5037 /* Caller is not willing to reclaim, we can't balance anything */ 5038 if (!can_direct_reclaim) 5039 goto nopage; 5040 5041 /* Avoid recursion of direct reclaim */ 5042 if (current->flags & PF_MEMALLOC) 5043 goto nopage; 5044 5045 /* Try direct reclaim and then allocating */ 5046 page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac, 5047 &did_some_progress); 5048 if (page) 5049 goto got_pg; 5050 5051 /* Try direct compaction and then allocating */ 5052 page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, ac, 5053 compact_priority, &compact_result); 5054 if (page) 5055 goto got_pg; 5056 5057 /* Do not loop if specifically requested */ 5058 if (gfp_mask & __GFP_NORETRY) 5059 goto nopage; 5060 5061 /* 5062 * Do not retry costly high order allocations unless they are 5063 * __GFP_RETRY_MAYFAIL 5064 */ 5065 if (costly_order && !(gfp_mask & __GFP_RETRY_MAYFAIL)) 5066 goto nopage; 5067 5068 if (should_reclaim_retry(gfp_mask, order, ac, alloc_flags, 5069 did_some_progress > 0, &no_progress_loops)) 5070 goto retry; 5071 5072 /* 5073 * It doesn't make any sense to retry for the compaction if the order-0 5074 * reclaim is not able to make any progress because the current 5075 * implementation of the compaction depends on the sufficient amount 5076 * of free memory (see __compaction_suitable) 5077 */ 5078 if (did_some_progress > 0 && 5079 should_compact_retry(ac, order, alloc_flags, 5080 compact_result, &compact_priority, 5081 &compaction_retries)) 5082 goto retry; 5083 5084 5085 /* Deal with possible cpuset update races before we start OOM killing */ 5086 if (check_retry_cpuset(cpuset_mems_cookie, ac)) 5087 goto retry_cpuset; 5088 5089 /* Reclaim has failed us, start killing things */ 5090 page = __alloc_pages_may_oom(gfp_mask, order, ac, &did_some_progress); 5091 if (page) 5092 goto got_pg; 5093 5094 /* Avoid allocations with no watermarks from looping endlessly */ 5095 if (tsk_is_oom_victim(current) && 5096 (alloc_flags & ALLOC_OOM || 5097 (gfp_mask & __GFP_NOMEMALLOC))) 5098 goto nopage; 5099 5100 /* Retry as long as the OOM killer is making progress */ 5101 if (did_some_progress) { 5102 no_progress_loops = 0; 5103 goto retry; 5104 } 5105 5106 nopage: 5107 /* Deal with possible cpuset update races before we fail */ 5108 if (check_retry_cpuset(cpuset_mems_cookie, ac)) 5109 goto retry_cpuset; 5110 5111 /* 5112 * Make sure that __GFP_NOFAIL request doesn't leak out and make sure 5113 * we always retry 5114 */ 5115 if (gfp_mask & __GFP_NOFAIL) { 5116 /* 5117 * All existing users of the __GFP_NOFAIL are blockable, so warn 5118 * of any new users that actually require GFP_NOWAIT 5119 */ 5120 if (WARN_ON_ONCE(!can_direct_reclaim)) 5121 goto fail; 5122 5123 /* 5124 * PF_MEMALLOC request from this context is rather bizarre 5125 * because we cannot reclaim anything and only can loop waiting 5126 * for somebody to do a work for us 5127 */ 5128 WARN_ON_ONCE(current->flags & PF_MEMALLOC); 5129 5130 /* 5131 * non failing costly orders are a hard requirement which we 5132 * are not prepared for much so let's warn about these users 5133 * so that we can identify them and convert them to something 5134 * else. 5135 */ 5136 WARN_ON_ONCE(order > PAGE_ALLOC_COSTLY_ORDER); 5137 5138 /* 5139 * Help non-failing allocations by giving them access to memory 5140 * reserves but do not use ALLOC_NO_WATERMARKS because this 5141 * could deplete whole memory reserves which would just make 5142 * the situation worse 5143 */ 5144 page = __alloc_pages_cpuset_fallback(gfp_mask, order, ALLOC_HARDER, ac); 5145 if (page) 5146 goto got_pg; 5147 5148 cond_resched(); 5149 goto retry; 5150 } 5151 fail: 5152 warn_alloc(gfp_mask, ac->nodemask, 5153 "page allocation failure: order:%u", order); 5154 got_pg: 5155 return page; 5156 } 5157 5158 static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order, 5159 int preferred_nid, nodemask_t *nodemask, 5160 struct alloc_context *ac, gfp_t *alloc_gfp, 5161 unsigned int *alloc_flags) 5162 { 5163 ac->highest_zoneidx = gfp_zone(gfp_mask); 5164 ac->zonelist = node_zonelist(preferred_nid, gfp_mask); 5165 ac->nodemask = nodemask; 5166 ac->migratetype = gfp_migratetype(gfp_mask); 5167 5168 if (cpusets_enabled()) { 5169 *alloc_gfp |= __GFP_HARDWALL; 5170 /* 5171 * When we are in the interrupt context, it is irrelevant 5172 * to the current task context. It means that any node ok. 5173 */ 5174 if (in_task() && !ac->nodemask) 5175 ac->nodemask = &cpuset_current_mems_allowed; 5176 else 5177 *alloc_flags |= ALLOC_CPUSET; 5178 } 5179 5180 fs_reclaim_acquire(gfp_mask); 5181 fs_reclaim_release(gfp_mask); 5182 5183 might_sleep_if(gfp_mask & __GFP_DIRECT_RECLAIM); 5184 5185 if (should_fail_alloc_page(gfp_mask, order)) 5186 return false; 5187 5188 *alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, *alloc_flags); 5189 5190 /* Dirty zone balancing only done in the fast path */ 5191 ac->spread_dirty_pages = (gfp_mask & __GFP_WRITE); 5192 5193 /* 5194 * The preferred zone is used for statistics but crucially it is 5195 * also used as the starting point for the zonelist iterator. It 5196 * may get reset for allocations that ignore memory policies. 5197 */ 5198 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist, 5199 ac->highest_zoneidx, ac->nodemask); 5200 5201 return true; 5202 } 5203 5204 /* 5205 * __alloc_pages_bulk - Allocate a number of order-0 pages to a list or array 5206 * @gfp: GFP flags for the allocation 5207 * @preferred_nid: The preferred NUMA node ID to allocate from 5208 * @nodemask: Set of nodes to allocate from, may be NULL 5209 * @nr_pages: The number of pages desired on the list or array 5210 * @page_list: Optional list to store the allocated pages 5211 * @page_array: Optional array to store the pages 5212 * 5213 * This is a batched version of the page allocator that attempts to 5214 * allocate nr_pages quickly. Pages are added to page_list if page_list 5215 * is not NULL, otherwise it is assumed that the page_array is valid. 5216 * 5217 * For lists, nr_pages is the number of pages that should be allocated. 5218 * 5219 * For arrays, only NULL elements are populated with pages and nr_pages 5220 * is the maximum number of pages that will be stored in the array. 5221 * 5222 * Returns the number of pages on the list or array. 5223 */ 5224 unsigned long __alloc_pages_bulk(gfp_t gfp, int preferred_nid, 5225 nodemask_t *nodemask, int nr_pages, 5226 struct list_head *page_list, 5227 struct page **page_array) 5228 { 5229 struct page *page; 5230 unsigned long flags; 5231 struct zone *zone; 5232 struct zoneref *z; 5233 struct per_cpu_pages *pcp; 5234 struct list_head *pcp_list; 5235 struct alloc_context ac; 5236 gfp_t alloc_gfp; 5237 unsigned int alloc_flags = ALLOC_WMARK_LOW; 5238 int nr_populated = 0, nr_account = 0; 5239 5240 /* 5241 * Skip populated array elements to determine if any pages need 5242 * to be allocated before disabling IRQs. 5243 */ 5244 while (page_array && nr_populated < nr_pages && page_array[nr_populated]) 5245 nr_populated++; 5246 5247 /* No pages requested? */ 5248 if (unlikely(nr_pages <= 0)) 5249 goto out; 5250 5251 /* Already populated array? */ 5252 if (unlikely(page_array && nr_pages - nr_populated == 0)) 5253 goto out; 5254 5255 /* Bulk allocator does not support memcg accounting. */ 5256 if (memcg_kmem_enabled() && (gfp & __GFP_ACCOUNT)) 5257 goto failed; 5258 5259 /* Use the single page allocator for one page. */ 5260 if (nr_pages - nr_populated == 1) 5261 goto failed; 5262 5263 #ifdef CONFIG_PAGE_OWNER 5264 /* 5265 * PAGE_OWNER may recurse into the allocator to allocate space to 5266 * save the stack with pagesets.lock held. Releasing/reacquiring 5267 * removes much of the performance benefit of bulk allocation so 5268 * force the caller to allocate one page at a time as it'll have 5269 * similar performance to added complexity to the bulk allocator. 5270 */ 5271 if (static_branch_unlikely(&page_owner_inited)) 5272 goto failed; 5273 #endif 5274 5275 /* May set ALLOC_NOFRAGMENT, fragmentation will return 1 page. */ 5276 gfp &= gfp_allowed_mask; 5277 alloc_gfp = gfp; 5278 if (!prepare_alloc_pages(gfp, 0, preferred_nid, nodemask, &ac, &alloc_gfp, &alloc_flags)) 5279 goto out; 5280 gfp = alloc_gfp; 5281 5282 /* Find an allowed local zone that meets the low watermark. */ 5283 for_each_zone_zonelist_nodemask(zone, z, ac.zonelist, ac.highest_zoneidx, ac.nodemask) { 5284 unsigned long mark; 5285 5286 if (cpusets_enabled() && (alloc_flags & ALLOC_CPUSET) && 5287 !__cpuset_zone_allowed(zone, gfp)) { 5288 continue; 5289 } 5290 5291 if (nr_online_nodes > 1 && zone != ac.preferred_zoneref->zone && 5292 zone_to_nid(zone) != zone_to_nid(ac.preferred_zoneref->zone)) { 5293 goto failed; 5294 } 5295 5296 mark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK) + nr_pages; 5297 if (zone_watermark_fast(zone, 0, mark, 5298 zonelist_zone_idx(ac.preferred_zoneref), 5299 alloc_flags, gfp)) { 5300 break; 5301 } 5302 } 5303 5304 /* 5305 * If there are no allowed local zones that meets the watermarks then 5306 * try to allocate a single page and reclaim if necessary. 5307 */ 5308 if (unlikely(!zone)) 5309 goto failed; 5310 5311 /* Attempt the batch allocation */ 5312 local_lock_irqsave(&pagesets.lock, flags); 5313 pcp = this_cpu_ptr(zone->per_cpu_pageset); 5314 pcp_list = &pcp->lists[order_to_pindex(ac.migratetype, 0)]; 5315 5316 while (nr_populated < nr_pages) { 5317 5318 /* Skip existing pages */ 5319 if (page_array && page_array[nr_populated]) { 5320 nr_populated++; 5321 continue; 5322 } 5323 5324 page = __rmqueue_pcplist(zone, 0, ac.migratetype, alloc_flags, 5325 pcp, pcp_list); 5326 if (unlikely(!page)) { 5327 /* Try and get at least one page */ 5328 if (!nr_populated) 5329 goto failed_irq; 5330 break; 5331 } 5332 nr_account++; 5333 5334 prep_new_page(page, 0, gfp, 0); 5335 if (page_list) 5336 list_add(&page->lru, page_list); 5337 else 5338 page_array[nr_populated] = page; 5339 nr_populated++; 5340 } 5341 5342 local_unlock_irqrestore(&pagesets.lock, flags); 5343 5344 __count_zid_vm_events(PGALLOC, zone_idx(zone), nr_account); 5345 zone_statistics(ac.preferred_zoneref->zone, zone, nr_account); 5346 5347 out: 5348 return nr_populated; 5349 5350 failed_irq: 5351 local_unlock_irqrestore(&pagesets.lock, flags); 5352 5353 failed: 5354 page = __alloc_pages(gfp, 0, preferred_nid, nodemask); 5355 if (page) { 5356 if (page_list) 5357 list_add(&page->lru, page_list); 5358 else 5359 page_array[nr_populated] = page; 5360 nr_populated++; 5361 } 5362 5363 goto out; 5364 } 5365 EXPORT_SYMBOL_GPL(__alloc_pages_bulk); 5366 5367 /* 5368 * This is the 'heart' of the zoned buddy allocator. 5369 */ 5370 struct page *__alloc_pages(gfp_t gfp, unsigned int order, int preferred_nid, 5371 nodemask_t *nodemask) 5372 { 5373 struct page *page; 5374 unsigned int alloc_flags = ALLOC_WMARK_LOW; 5375 gfp_t alloc_gfp; /* The gfp_t that was actually used for allocation */ 5376 struct alloc_context ac = { }; 5377 5378 /* 5379 * There are several places where we assume that the order value is sane 5380 * so bail out early if the request is out of bound. 5381 */ 5382 if (unlikely(order >= MAX_ORDER)) { 5383 WARN_ON_ONCE(!(gfp & __GFP_NOWARN)); 5384 return NULL; 5385 } 5386 5387 gfp &= gfp_allowed_mask; 5388 /* 5389 * Apply scoped allocation constraints. This is mainly about GFP_NOFS 5390 * resp. GFP_NOIO which has to be inherited for all allocation requests 5391 * from a particular context which has been marked by 5392 * memalloc_no{fs,io}_{save,restore}. And PF_MEMALLOC_PIN which ensures 5393 * movable zones are not used during allocation. 5394 */ 5395 gfp = current_gfp_context(gfp); 5396 alloc_gfp = gfp; 5397 if (!prepare_alloc_pages(gfp, order, preferred_nid, nodemask, &ac, 5398 &alloc_gfp, &alloc_flags)) 5399 return NULL; 5400 5401 /* 5402 * Forbid the first pass from falling back to types that fragment 5403 * memory until all local zones are considered. 5404 */ 5405 alloc_flags |= alloc_flags_nofragment(ac.preferred_zoneref->zone, gfp); 5406 5407 /* First allocation attempt */ 5408 page = get_page_from_freelist(alloc_gfp, order, alloc_flags, &ac); 5409 if (likely(page)) 5410 goto out; 5411 5412 alloc_gfp = gfp; 5413 ac.spread_dirty_pages = false; 5414 5415 /* 5416 * Restore the original nodemask if it was potentially replaced with 5417 * &cpuset_current_mems_allowed to optimize the fast-path attempt. 5418 */ 5419 ac.nodemask = nodemask; 5420 5421 page = __alloc_pages_slowpath(alloc_gfp, order, &ac); 5422 5423 out: 5424 if (memcg_kmem_enabled() && (gfp & __GFP_ACCOUNT) && page && 5425 unlikely(__memcg_kmem_charge_page(page, gfp, order) != 0)) { 5426 __free_pages(page, order); 5427 page = NULL; 5428 } 5429 5430 trace_mm_page_alloc(page, order, alloc_gfp, ac.migratetype); 5431 5432 return page; 5433 } 5434 EXPORT_SYMBOL(__alloc_pages); 5435 5436 struct folio *__folio_alloc(gfp_t gfp, unsigned int order, int preferred_nid, 5437 nodemask_t *nodemask) 5438 { 5439 struct page *page = __alloc_pages(gfp | __GFP_COMP, order, 5440 preferred_nid, nodemask); 5441 5442 if (page && order > 1) 5443 prep_transhuge_page(page); 5444 return (struct folio *)page; 5445 } 5446 EXPORT_SYMBOL(__folio_alloc); 5447 5448 /* 5449 * Common helper functions. Never use with __GFP_HIGHMEM because the returned 5450 * address cannot represent highmem pages. Use alloc_pages and then kmap if 5451 * you need to access high mem. 5452 */ 5453 unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order) 5454 { 5455 struct page *page; 5456 5457 page = alloc_pages(gfp_mask & ~__GFP_HIGHMEM, order); 5458 if (!page) 5459 return 0; 5460 return (unsigned long) page_address(page); 5461 } 5462 EXPORT_SYMBOL(__get_free_pages); 5463 5464 unsigned long get_zeroed_page(gfp_t gfp_mask) 5465 { 5466 return __get_free_pages(gfp_mask | __GFP_ZERO, 0); 5467 } 5468 EXPORT_SYMBOL(get_zeroed_page); 5469 5470 /** 5471 * __free_pages - Free pages allocated with alloc_pages(). 5472 * @page: The page pointer returned from alloc_pages(). 5473 * @order: The order of the allocation. 5474 * 5475 * This function can free multi-page allocations that are not compound 5476 * pages. It does not check that the @order passed in matches that of 5477 * the allocation, so it is easy to leak memory. Freeing more memory 5478 * than was allocated will probably emit a warning. 5479 * 5480 * If the last reference to this page is speculative, it will be released 5481 * by put_page() which only frees the first page of a non-compound 5482 * allocation. To prevent the remaining pages from being leaked, we free 5483 * the subsequent pages here. If you want to use the page's reference 5484 * count to decide when to free the allocation, you should allocate a 5485 * compound page, and use put_page() instead of __free_pages(). 5486 * 5487 * Context: May be called in interrupt context or while holding a normal 5488 * spinlock, but not in NMI context or while holding a raw spinlock. 5489 */ 5490 void __free_pages(struct page *page, unsigned int order) 5491 { 5492 if (put_page_testzero(page)) 5493 free_the_page(page, order); 5494 else if (!PageHead(page)) 5495 while (order-- > 0) 5496 free_the_page(page + (1 << order), order); 5497 } 5498 EXPORT_SYMBOL(__free_pages); 5499 5500 void free_pages(unsigned long addr, unsigned int order) 5501 { 5502 if (addr != 0) { 5503 VM_BUG_ON(!virt_addr_valid((void *)addr)); 5504 __free_pages(virt_to_page((void *)addr), order); 5505 } 5506 } 5507 5508 EXPORT_SYMBOL(free_pages); 5509 5510 /* 5511 * Page Fragment: 5512 * An arbitrary-length arbitrary-offset area of memory which resides 5513 * within a 0 or higher order page. Multiple fragments within that page 5514 * are individually refcounted, in the page's reference counter. 5515 * 5516 * The page_frag functions below provide a simple allocation framework for 5517 * page fragments. This is used by the network stack and network device 5518 * drivers to provide a backing region of memory for use as either an 5519 * sk_buff->head, or to be used in the "frags" portion of skb_shared_info. 5520 */ 5521 static struct page *__page_frag_cache_refill(struct page_frag_cache *nc, 5522 gfp_t gfp_mask) 5523 { 5524 struct page *page = NULL; 5525 gfp_t gfp = gfp_mask; 5526 5527 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 5528 gfp_mask |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY | 5529 __GFP_NOMEMALLOC; 5530 page = alloc_pages_node(NUMA_NO_NODE, gfp_mask, 5531 PAGE_FRAG_CACHE_MAX_ORDER); 5532 nc->size = page ? PAGE_FRAG_CACHE_MAX_SIZE : PAGE_SIZE; 5533 #endif 5534 if (unlikely(!page)) 5535 page = alloc_pages_node(NUMA_NO_NODE, gfp, 0); 5536 5537 nc->va = page ? page_address(page) : NULL; 5538 5539 return page; 5540 } 5541 5542 void __page_frag_cache_drain(struct page *page, unsigned int count) 5543 { 5544 VM_BUG_ON_PAGE(page_ref_count(page) == 0, page); 5545 5546 if (page_ref_sub_and_test(page, count)) 5547 free_the_page(page, compound_order(page)); 5548 } 5549 EXPORT_SYMBOL(__page_frag_cache_drain); 5550 5551 void *page_frag_alloc_align(struct page_frag_cache *nc, 5552 unsigned int fragsz, gfp_t gfp_mask, 5553 unsigned int align_mask) 5554 { 5555 unsigned int size = PAGE_SIZE; 5556 struct page *page; 5557 int offset; 5558 5559 if (unlikely(!nc->va)) { 5560 refill: 5561 page = __page_frag_cache_refill(nc, gfp_mask); 5562 if (!page) 5563 return NULL; 5564 5565 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 5566 /* if size can vary use size else just use PAGE_SIZE */ 5567 size = nc->size; 5568 #endif 5569 /* Even if we own the page, we do not use atomic_set(). 5570 * This would break get_page_unless_zero() users. 5571 */ 5572 page_ref_add(page, PAGE_FRAG_CACHE_MAX_SIZE); 5573 5574 /* reset page count bias and offset to start of new frag */ 5575 nc->pfmemalloc = page_is_pfmemalloc(page); 5576 nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1; 5577 nc->offset = size; 5578 } 5579 5580 offset = nc->offset - fragsz; 5581 if (unlikely(offset < 0)) { 5582 page = virt_to_page(nc->va); 5583 5584 if (!page_ref_sub_and_test(page, nc->pagecnt_bias)) 5585 goto refill; 5586 5587 if (unlikely(nc->pfmemalloc)) { 5588 free_the_page(page, compound_order(page)); 5589 goto refill; 5590 } 5591 5592 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 5593 /* if size can vary use size else just use PAGE_SIZE */ 5594 size = nc->size; 5595 #endif 5596 /* OK, page count is 0, we can safely set it */ 5597 set_page_count(page, PAGE_FRAG_CACHE_MAX_SIZE + 1); 5598 5599 /* reset page count bias and offset to start of new frag */ 5600 nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1; 5601 offset = size - fragsz; 5602 } 5603 5604 nc->pagecnt_bias--; 5605 offset &= align_mask; 5606 nc->offset = offset; 5607 5608 return nc->va + offset; 5609 } 5610 EXPORT_SYMBOL(page_frag_alloc_align); 5611 5612 /* 5613 * Frees a page fragment allocated out of either a compound or order 0 page. 5614 */ 5615 void page_frag_free(void *addr) 5616 { 5617 struct page *page = virt_to_head_page(addr); 5618 5619 if (unlikely(put_page_testzero(page))) 5620 free_the_page(page, compound_order(page)); 5621 } 5622 EXPORT_SYMBOL(page_frag_free); 5623 5624 static void *make_alloc_exact(unsigned long addr, unsigned int order, 5625 size_t size) 5626 { 5627 if (addr) { 5628 unsigned long alloc_end = addr + (PAGE_SIZE << order); 5629 unsigned long used = addr + PAGE_ALIGN(size); 5630 5631 split_page(virt_to_page((void *)addr), order); 5632 while (used < alloc_end) { 5633 free_page(used); 5634 used += PAGE_SIZE; 5635 } 5636 } 5637 return (void *)addr; 5638 } 5639 5640 /** 5641 * alloc_pages_exact - allocate an exact number physically-contiguous pages. 5642 * @size: the number of bytes to allocate 5643 * @gfp_mask: GFP flags for the allocation, must not contain __GFP_COMP 5644 * 5645 * This function is similar to alloc_pages(), except that it allocates the 5646 * minimum number of pages to satisfy the request. alloc_pages() can only 5647 * allocate memory in power-of-two pages. 5648 * 5649 * This function is also limited by MAX_ORDER. 5650 * 5651 * Memory allocated by this function must be released by free_pages_exact(). 5652 * 5653 * Return: pointer to the allocated area or %NULL in case of error. 5654 */ 5655 void *alloc_pages_exact(size_t size, gfp_t gfp_mask) 5656 { 5657 unsigned int order = get_order(size); 5658 unsigned long addr; 5659 5660 if (WARN_ON_ONCE(gfp_mask & (__GFP_COMP | __GFP_HIGHMEM))) 5661 gfp_mask &= ~(__GFP_COMP | __GFP_HIGHMEM); 5662 5663 addr = __get_free_pages(gfp_mask, order); 5664 return make_alloc_exact(addr, order, size); 5665 } 5666 EXPORT_SYMBOL(alloc_pages_exact); 5667 5668 /** 5669 * alloc_pages_exact_nid - allocate an exact number of physically-contiguous 5670 * pages on a node. 5671 * @nid: the preferred node ID where memory should be allocated 5672 * @size: the number of bytes to allocate 5673 * @gfp_mask: GFP flags for the allocation, must not contain __GFP_COMP 5674 * 5675 * Like alloc_pages_exact(), but try to allocate on node nid first before falling 5676 * back. 5677 * 5678 * Return: pointer to the allocated area or %NULL in case of error. 5679 */ 5680 void * __meminit alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask) 5681 { 5682 unsigned int order = get_order(size); 5683 struct page *p; 5684 5685 if (WARN_ON_ONCE(gfp_mask & (__GFP_COMP | __GFP_HIGHMEM))) 5686 gfp_mask &= ~(__GFP_COMP | __GFP_HIGHMEM); 5687 5688 p = alloc_pages_node(nid, gfp_mask, order); 5689 if (!p) 5690 return NULL; 5691 return make_alloc_exact((unsigned long)page_address(p), order, size); 5692 } 5693 5694 /** 5695 * free_pages_exact - release memory allocated via alloc_pages_exact() 5696 * @virt: the value returned by alloc_pages_exact. 5697 * @size: size of allocation, same value as passed to alloc_pages_exact(). 5698 * 5699 * Release the memory allocated by a previous call to alloc_pages_exact. 5700 */ 5701 void free_pages_exact(void *virt, size_t size) 5702 { 5703 unsigned long addr = (unsigned long)virt; 5704 unsigned long end = addr + PAGE_ALIGN(size); 5705 5706 while (addr < end) { 5707 free_page(addr); 5708 addr += PAGE_SIZE; 5709 } 5710 } 5711 EXPORT_SYMBOL(free_pages_exact); 5712 5713 /** 5714 * nr_free_zone_pages - count number of pages beyond high watermark 5715 * @offset: The zone index of the highest zone 5716 * 5717 * nr_free_zone_pages() counts the number of pages which are beyond the 5718 * high watermark within all zones at or below a given zone index. For each 5719 * zone, the number of pages is calculated as: 5720 * 5721 * nr_free_zone_pages = managed_pages - high_pages 5722 * 5723 * Return: number of pages beyond high watermark. 5724 */ 5725 static unsigned long nr_free_zone_pages(int offset) 5726 { 5727 struct zoneref *z; 5728 struct zone *zone; 5729 5730 /* Just pick one node, since fallback list is circular */ 5731 unsigned long sum = 0; 5732 5733 struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL); 5734 5735 for_each_zone_zonelist(zone, z, zonelist, offset) { 5736 unsigned long size = zone_managed_pages(zone); 5737 unsigned long high = high_wmark_pages(zone); 5738 if (size > high) 5739 sum += size - high; 5740 } 5741 5742 return sum; 5743 } 5744 5745 /** 5746 * nr_free_buffer_pages - count number of pages beyond high watermark 5747 * 5748 * nr_free_buffer_pages() counts the number of pages which are beyond the high 5749 * watermark within ZONE_DMA and ZONE_NORMAL. 5750 * 5751 * Return: number of pages beyond high watermark within ZONE_DMA and 5752 * ZONE_NORMAL. 5753 */ 5754 unsigned long nr_free_buffer_pages(void) 5755 { 5756 return nr_free_zone_pages(gfp_zone(GFP_USER)); 5757 } 5758 EXPORT_SYMBOL_GPL(nr_free_buffer_pages); 5759 5760 static inline void show_node(struct zone *zone) 5761 { 5762 if (IS_ENABLED(CONFIG_NUMA)) 5763 printk("Node %d ", zone_to_nid(zone)); 5764 } 5765 5766 long si_mem_available(void) 5767 { 5768 long available; 5769 unsigned long pagecache; 5770 unsigned long wmark_low = 0; 5771 unsigned long pages[NR_LRU_LISTS]; 5772 unsigned long reclaimable; 5773 struct zone *zone; 5774 int lru; 5775 5776 for (lru = LRU_BASE; lru < NR_LRU_LISTS; lru++) 5777 pages[lru] = global_node_page_state(NR_LRU_BASE + lru); 5778 5779 for_each_zone(zone) 5780 wmark_low += low_wmark_pages(zone); 5781 5782 /* 5783 * Estimate the amount of memory available for userspace allocations, 5784 * without causing swapping. 5785 */ 5786 available = global_zone_page_state(NR_FREE_PAGES) - totalreserve_pages; 5787 5788 /* 5789 * Not all the page cache can be freed, otherwise the system will 5790 * start swapping. Assume at least half of the page cache, or the 5791 * low watermark worth of cache, needs to stay. 5792 */ 5793 pagecache = pages[LRU_ACTIVE_FILE] + pages[LRU_INACTIVE_FILE]; 5794 pagecache -= min(pagecache / 2, wmark_low); 5795 available += pagecache; 5796 5797 /* 5798 * Part of the reclaimable slab and other kernel memory consists of 5799 * items that are in use, and cannot be freed. Cap this estimate at the 5800 * low watermark. 5801 */ 5802 reclaimable = global_node_page_state_pages(NR_SLAB_RECLAIMABLE_B) + 5803 global_node_page_state(NR_KERNEL_MISC_RECLAIMABLE); 5804 available += reclaimable - min(reclaimable / 2, wmark_low); 5805 5806 if (available < 0) 5807 available = 0; 5808 return available; 5809 } 5810 EXPORT_SYMBOL_GPL(si_mem_available); 5811 5812 void si_meminfo(struct sysinfo *val) 5813 { 5814 val->totalram = totalram_pages(); 5815 val->sharedram = global_node_page_state(NR_SHMEM); 5816 val->freeram = global_zone_page_state(NR_FREE_PAGES); 5817 val->bufferram = nr_blockdev_pages(); 5818 val->totalhigh = totalhigh_pages(); 5819 val->freehigh = nr_free_highpages(); 5820 val->mem_unit = PAGE_SIZE; 5821 } 5822 5823 EXPORT_SYMBOL(si_meminfo); 5824 5825 #ifdef CONFIG_NUMA 5826 void si_meminfo_node(struct sysinfo *val, int nid) 5827 { 5828 int zone_type; /* needs to be signed */ 5829 unsigned long managed_pages = 0; 5830 unsigned long managed_highpages = 0; 5831 unsigned long free_highpages = 0; 5832 pg_data_t *pgdat = NODE_DATA(nid); 5833 5834 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) 5835 managed_pages += zone_managed_pages(&pgdat->node_zones[zone_type]); 5836 val->totalram = managed_pages; 5837 val->sharedram = node_page_state(pgdat, NR_SHMEM); 5838 val->freeram = sum_zone_node_page_state(nid, NR_FREE_PAGES); 5839 #ifdef CONFIG_HIGHMEM 5840 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) { 5841 struct zone *zone = &pgdat->node_zones[zone_type]; 5842 5843 if (is_highmem(zone)) { 5844 managed_highpages += zone_managed_pages(zone); 5845 free_highpages += zone_page_state(zone, NR_FREE_PAGES); 5846 } 5847 } 5848 val->totalhigh = managed_highpages; 5849 val->freehigh = free_highpages; 5850 #else 5851 val->totalhigh = managed_highpages; 5852 val->freehigh = free_highpages; 5853 #endif 5854 val->mem_unit = PAGE_SIZE; 5855 } 5856 #endif 5857 5858 /* 5859 * Determine whether the node should be displayed or not, depending on whether 5860 * SHOW_MEM_FILTER_NODES was passed to show_free_areas(). 5861 */ 5862 static bool show_mem_node_skip(unsigned int flags, int nid, nodemask_t *nodemask) 5863 { 5864 if (!(flags & SHOW_MEM_FILTER_NODES)) 5865 return false; 5866 5867 /* 5868 * no node mask - aka implicit memory numa policy. Do not bother with 5869 * the synchronization - read_mems_allowed_begin - because we do not 5870 * have to be precise here. 5871 */ 5872 if (!nodemask) 5873 nodemask = &cpuset_current_mems_allowed; 5874 5875 return !node_isset(nid, *nodemask); 5876 } 5877 5878 #define K(x) ((x) << (PAGE_SHIFT-10)) 5879 5880 static void show_migration_types(unsigned char type) 5881 { 5882 static const char types[MIGRATE_TYPES] = { 5883 [MIGRATE_UNMOVABLE] = 'U', 5884 [MIGRATE_MOVABLE] = 'M', 5885 [MIGRATE_RECLAIMABLE] = 'E', 5886 [MIGRATE_HIGHATOMIC] = 'H', 5887 #ifdef CONFIG_CMA 5888 [MIGRATE_CMA] = 'C', 5889 #endif 5890 #ifdef CONFIG_MEMORY_ISOLATION 5891 [MIGRATE_ISOLATE] = 'I', 5892 #endif 5893 }; 5894 char tmp[MIGRATE_TYPES + 1]; 5895 char *p = tmp; 5896 int i; 5897 5898 for (i = 0; i < MIGRATE_TYPES; i++) { 5899 if (type & (1 << i)) 5900 *p++ = types[i]; 5901 } 5902 5903 *p = '\0'; 5904 printk(KERN_CONT "(%s) ", tmp); 5905 } 5906 5907 /* 5908 * Show free area list (used inside shift_scroll-lock stuff) 5909 * We also calculate the percentage fragmentation. We do this by counting the 5910 * memory on each free list with the exception of the first item on the list. 5911 * 5912 * Bits in @filter: 5913 * SHOW_MEM_FILTER_NODES: suppress nodes that are not allowed by current's 5914 * cpuset. 5915 */ 5916 void show_free_areas(unsigned int filter, nodemask_t *nodemask) 5917 { 5918 unsigned long free_pcp = 0; 5919 int cpu; 5920 struct zone *zone; 5921 pg_data_t *pgdat; 5922 5923 for_each_populated_zone(zone) { 5924 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask)) 5925 continue; 5926 5927 for_each_online_cpu(cpu) 5928 free_pcp += per_cpu_ptr(zone->per_cpu_pageset, cpu)->count; 5929 } 5930 5931 printk("active_anon:%lu inactive_anon:%lu isolated_anon:%lu\n" 5932 " active_file:%lu inactive_file:%lu isolated_file:%lu\n" 5933 " unevictable:%lu dirty:%lu writeback:%lu\n" 5934 " slab_reclaimable:%lu slab_unreclaimable:%lu\n" 5935 " mapped:%lu shmem:%lu pagetables:%lu bounce:%lu\n" 5936 " kernel_misc_reclaimable:%lu\n" 5937 " free:%lu free_pcp:%lu free_cma:%lu\n", 5938 global_node_page_state(NR_ACTIVE_ANON), 5939 global_node_page_state(NR_INACTIVE_ANON), 5940 global_node_page_state(NR_ISOLATED_ANON), 5941 global_node_page_state(NR_ACTIVE_FILE), 5942 global_node_page_state(NR_INACTIVE_FILE), 5943 global_node_page_state(NR_ISOLATED_FILE), 5944 global_node_page_state(NR_UNEVICTABLE), 5945 global_node_page_state(NR_FILE_DIRTY), 5946 global_node_page_state(NR_WRITEBACK), 5947 global_node_page_state_pages(NR_SLAB_RECLAIMABLE_B), 5948 global_node_page_state_pages(NR_SLAB_UNRECLAIMABLE_B), 5949 global_node_page_state(NR_FILE_MAPPED), 5950 global_node_page_state(NR_SHMEM), 5951 global_node_page_state(NR_PAGETABLE), 5952 global_zone_page_state(NR_BOUNCE), 5953 global_node_page_state(NR_KERNEL_MISC_RECLAIMABLE), 5954 global_zone_page_state(NR_FREE_PAGES), 5955 free_pcp, 5956 global_zone_page_state(NR_FREE_CMA_PAGES)); 5957 5958 for_each_online_pgdat(pgdat) { 5959 if (show_mem_node_skip(filter, pgdat->node_id, nodemask)) 5960 continue; 5961 5962 printk("Node %d" 5963 " active_anon:%lukB" 5964 " inactive_anon:%lukB" 5965 " active_file:%lukB" 5966 " inactive_file:%lukB" 5967 " unevictable:%lukB" 5968 " isolated(anon):%lukB" 5969 " isolated(file):%lukB" 5970 " mapped:%lukB" 5971 " dirty:%lukB" 5972 " writeback:%lukB" 5973 " shmem:%lukB" 5974 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 5975 " shmem_thp: %lukB" 5976 " shmem_pmdmapped: %lukB" 5977 " anon_thp: %lukB" 5978 #endif 5979 " writeback_tmp:%lukB" 5980 " kernel_stack:%lukB" 5981 #ifdef CONFIG_SHADOW_CALL_STACK 5982 " shadow_call_stack:%lukB" 5983 #endif 5984 " pagetables:%lukB" 5985 " all_unreclaimable? %s" 5986 "\n", 5987 pgdat->node_id, 5988 K(node_page_state(pgdat, NR_ACTIVE_ANON)), 5989 K(node_page_state(pgdat, NR_INACTIVE_ANON)), 5990 K(node_page_state(pgdat, NR_ACTIVE_FILE)), 5991 K(node_page_state(pgdat, NR_INACTIVE_FILE)), 5992 K(node_page_state(pgdat, NR_UNEVICTABLE)), 5993 K(node_page_state(pgdat, NR_ISOLATED_ANON)), 5994 K(node_page_state(pgdat, NR_ISOLATED_FILE)), 5995 K(node_page_state(pgdat, NR_FILE_MAPPED)), 5996 K(node_page_state(pgdat, NR_FILE_DIRTY)), 5997 K(node_page_state(pgdat, NR_WRITEBACK)), 5998 K(node_page_state(pgdat, NR_SHMEM)), 5999 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 6000 K(node_page_state(pgdat, NR_SHMEM_THPS)), 6001 K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED)), 6002 K(node_page_state(pgdat, NR_ANON_THPS)), 6003 #endif 6004 K(node_page_state(pgdat, NR_WRITEBACK_TEMP)), 6005 node_page_state(pgdat, NR_KERNEL_STACK_KB), 6006 #ifdef CONFIG_SHADOW_CALL_STACK 6007 node_page_state(pgdat, NR_KERNEL_SCS_KB), 6008 #endif 6009 K(node_page_state(pgdat, NR_PAGETABLE)), 6010 pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES ? 6011 "yes" : "no"); 6012 } 6013 6014 for_each_populated_zone(zone) { 6015 int i; 6016 6017 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask)) 6018 continue; 6019 6020 free_pcp = 0; 6021 for_each_online_cpu(cpu) 6022 free_pcp += per_cpu_ptr(zone->per_cpu_pageset, cpu)->count; 6023 6024 show_node(zone); 6025 printk(KERN_CONT 6026 "%s" 6027 " free:%lukB" 6028 " boost:%lukB" 6029 " min:%lukB" 6030 " low:%lukB" 6031 " high:%lukB" 6032 " reserved_highatomic:%luKB" 6033 " active_anon:%lukB" 6034 " inactive_anon:%lukB" 6035 " active_file:%lukB" 6036 " inactive_file:%lukB" 6037 " unevictable:%lukB" 6038 " writepending:%lukB" 6039 " present:%lukB" 6040 " managed:%lukB" 6041 " mlocked:%lukB" 6042 " bounce:%lukB" 6043 " free_pcp:%lukB" 6044 " local_pcp:%ukB" 6045 " free_cma:%lukB" 6046 "\n", 6047 zone->name, 6048 K(zone_page_state(zone, NR_FREE_PAGES)), 6049 K(zone->watermark_boost), 6050 K(min_wmark_pages(zone)), 6051 K(low_wmark_pages(zone)), 6052 K(high_wmark_pages(zone)), 6053 K(zone->nr_reserved_highatomic), 6054 K(zone_page_state(zone, NR_ZONE_ACTIVE_ANON)), 6055 K(zone_page_state(zone, NR_ZONE_INACTIVE_ANON)), 6056 K(zone_page_state(zone, NR_ZONE_ACTIVE_FILE)), 6057 K(zone_page_state(zone, NR_ZONE_INACTIVE_FILE)), 6058 K(zone_page_state(zone, NR_ZONE_UNEVICTABLE)), 6059 K(zone_page_state(zone, NR_ZONE_WRITE_PENDING)), 6060 K(zone->present_pages), 6061 K(zone_managed_pages(zone)), 6062 K(zone_page_state(zone, NR_MLOCK)), 6063 K(zone_page_state(zone, NR_BOUNCE)), 6064 K(free_pcp), 6065 K(this_cpu_read(zone->per_cpu_pageset->count)), 6066 K(zone_page_state(zone, NR_FREE_CMA_PAGES))); 6067 printk("lowmem_reserve[]:"); 6068 for (i = 0; i < MAX_NR_ZONES; i++) 6069 printk(KERN_CONT " %ld", zone->lowmem_reserve[i]); 6070 printk(KERN_CONT "\n"); 6071 } 6072 6073 for_each_populated_zone(zone) { 6074 unsigned int order; 6075 unsigned long nr[MAX_ORDER], flags, total = 0; 6076 unsigned char types[MAX_ORDER]; 6077 6078 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask)) 6079 continue; 6080 show_node(zone); 6081 printk(KERN_CONT "%s: ", zone->name); 6082 6083 spin_lock_irqsave(&zone->lock, flags); 6084 for (order = 0; order < MAX_ORDER; order++) { 6085 struct free_area *area = &zone->free_area[order]; 6086 int type; 6087 6088 nr[order] = area->nr_free; 6089 total += nr[order] << order; 6090 6091 types[order] = 0; 6092 for (type = 0; type < MIGRATE_TYPES; type++) { 6093 if (!free_area_empty(area, type)) 6094 types[order] |= 1 << type; 6095 } 6096 } 6097 spin_unlock_irqrestore(&zone->lock, flags); 6098 for (order = 0; order < MAX_ORDER; order++) { 6099 printk(KERN_CONT "%lu*%lukB ", 6100 nr[order], K(1UL) << order); 6101 if (nr[order]) 6102 show_migration_types(types[order]); 6103 } 6104 printk(KERN_CONT "= %lukB\n", K(total)); 6105 } 6106 6107 hugetlb_show_meminfo(); 6108 6109 printk("%ld total pagecache pages\n", global_node_page_state(NR_FILE_PAGES)); 6110 6111 show_swap_cache_info(); 6112 } 6113 6114 static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref) 6115 { 6116 zoneref->zone = zone; 6117 zoneref->zone_idx = zone_idx(zone); 6118 } 6119 6120 /* 6121 * Builds allocation fallback zone lists. 6122 * 6123 * Add all populated zones of a node to the zonelist. 6124 */ 6125 static int build_zonerefs_node(pg_data_t *pgdat, struct zoneref *zonerefs) 6126 { 6127 struct zone *zone; 6128 enum zone_type zone_type = MAX_NR_ZONES; 6129 int nr_zones = 0; 6130 6131 do { 6132 zone_type--; 6133 zone = pgdat->node_zones + zone_type; 6134 if (populated_zone(zone)) { 6135 zoneref_set_zone(zone, &zonerefs[nr_zones++]); 6136 check_highest_zone(zone_type); 6137 } 6138 } while (zone_type); 6139 6140 return nr_zones; 6141 } 6142 6143 #ifdef CONFIG_NUMA 6144 6145 static int __parse_numa_zonelist_order(char *s) 6146 { 6147 /* 6148 * We used to support different zonelists modes but they turned 6149 * out to be just not useful. Let's keep the warning in place 6150 * if somebody still use the cmd line parameter so that we do 6151 * not fail it silently 6152 */ 6153 if (!(*s == 'd' || *s == 'D' || *s == 'n' || *s == 'N')) { 6154 pr_warn("Ignoring unsupported numa_zonelist_order value: %s\n", s); 6155 return -EINVAL; 6156 } 6157 return 0; 6158 } 6159 6160 char numa_zonelist_order[] = "Node"; 6161 6162 /* 6163 * sysctl handler for numa_zonelist_order 6164 */ 6165 int numa_zonelist_order_handler(struct ctl_table *table, int write, 6166 void *buffer, size_t *length, loff_t *ppos) 6167 { 6168 if (write) 6169 return __parse_numa_zonelist_order(buffer); 6170 return proc_dostring(table, write, buffer, length, ppos); 6171 } 6172 6173 6174 #define MAX_NODE_LOAD (nr_online_nodes) 6175 static int node_load[MAX_NUMNODES]; 6176 6177 /** 6178 * find_next_best_node - find the next node that should appear in a given node's fallback list 6179 * @node: node whose fallback list we're appending 6180 * @used_node_mask: nodemask_t of already used nodes 6181 * 6182 * We use a number of factors to determine which is the next node that should 6183 * appear on a given node's fallback list. The node should not have appeared 6184 * already in @node's fallback list, and it should be the next closest node 6185 * according to the distance array (which contains arbitrary distance values 6186 * from each node to each node in the system), and should also prefer nodes 6187 * with no CPUs, since presumably they'll have very little allocation pressure 6188 * on them otherwise. 6189 * 6190 * Return: node id of the found node or %NUMA_NO_NODE if no node is found. 6191 */ 6192 int find_next_best_node(int node, nodemask_t *used_node_mask) 6193 { 6194 int n, val; 6195 int min_val = INT_MAX; 6196 int best_node = NUMA_NO_NODE; 6197 6198 /* Use the local node if we haven't already */ 6199 if (!node_isset(node, *used_node_mask)) { 6200 node_set(node, *used_node_mask); 6201 return node; 6202 } 6203 6204 for_each_node_state(n, N_MEMORY) { 6205 6206 /* Don't want a node to appear more than once */ 6207 if (node_isset(n, *used_node_mask)) 6208 continue; 6209 6210 /* Use the distance array to find the distance */ 6211 val = node_distance(node, n); 6212 6213 /* Penalize nodes under us ("prefer the next node") */ 6214 val += (n < node); 6215 6216 /* Give preference to headless and unused nodes */ 6217 if (!cpumask_empty(cpumask_of_node(n))) 6218 val += PENALTY_FOR_NODE_WITH_CPUS; 6219 6220 /* Slight preference for less loaded node */ 6221 val *= (MAX_NODE_LOAD*MAX_NUMNODES); 6222 val += node_load[n]; 6223 6224 if (val < min_val) { 6225 min_val = val; 6226 best_node = n; 6227 } 6228 } 6229 6230 if (best_node >= 0) 6231 node_set(best_node, *used_node_mask); 6232 6233 return best_node; 6234 } 6235 6236 6237 /* 6238 * Build zonelists ordered by node and zones within node. 6239 * This results in maximum locality--normal zone overflows into local 6240 * DMA zone, if any--but risks exhausting DMA zone. 6241 */ 6242 static void build_zonelists_in_node_order(pg_data_t *pgdat, int *node_order, 6243 unsigned nr_nodes) 6244 { 6245 struct zoneref *zonerefs; 6246 int i; 6247 6248 zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs; 6249 6250 for (i = 0; i < nr_nodes; i++) { 6251 int nr_zones; 6252 6253 pg_data_t *node = NODE_DATA(node_order[i]); 6254 6255 nr_zones = build_zonerefs_node(node, zonerefs); 6256 zonerefs += nr_zones; 6257 } 6258 zonerefs->zone = NULL; 6259 zonerefs->zone_idx = 0; 6260 } 6261 6262 /* 6263 * Build gfp_thisnode zonelists 6264 */ 6265 static void build_thisnode_zonelists(pg_data_t *pgdat) 6266 { 6267 struct zoneref *zonerefs; 6268 int nr_zones; 6269 6270 zonerefs = pgdat->node_zonelists[ZONELIST_NOFALLBACK]._zonerefs; 6271 nr_zones = build_zonerefs_node(pgdat, zonerefs); 6272 zonerefs += nr_zones; 6273 zonerefs->zone = NULL; 6274 zonerefs->zone_idx = 0; 6275 } 6276 6277 /* 6278 * Build zonelists ordered by zone and nodes within zones. 6279 * This results in conserving DMA zone[s] until all Normal memory is 6280 * exhausted, but results in overflowing to remote node while memory 6281 * may still exist in local DMA zone. 6282 */ 6283 6284 static void build_zonelists(pg_data_t *pgdat) 6285 { 6286 static int node_order[MAX_NUMNODES]; 6287 int node, load, nr_nodes = 0; 6288 nodemask_t used_mask = NODE_MASK_NONE; 6289 int local_node, prev_node; 6290 6291 /* NUMA-aware ordering of nodes */ 6292 local_node = pgdat->node_id; 6293 load = nr_online_nodes; 6294 prev_node = local_node; 6295 6296 memset(node_order, 0, sizeof(node_order)); 6297 while ((node = find_next_best_node(local_node, &used_mask)) >= 0) { 6298 /* 6299 * We don't want to pressure a particular node. 6300 * So adding penalty to the first node in same 6301 * distance group to make it round-robin. 6302 */ 6303 if (node_distance(local_node, node) != 6304 node_distance(local_node, prev_node)) 6305 node_load[node] += load; 6306 6307 node_order[nr_nodes++] = node; 6308 prev_node = node; 6309 load--; 6310 } 6311 6312 build_zonelists_in_node_order(pgdat, node_order, nr_nodes); 6313 build_thisnode_zonelists(pgdat); 6314 pr_info("Fallback order for Node %d: ", local_node); 6315 for (node = 0; node < nr_nodes; node++) 6316 pr_cont("%d ", node_order[node]); 6317 pr_cont("\n"); 6318 } 6319 6320 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 6321 /* 6322 * Return node id of node used for "local" allocations. 6323 * I.e., first node id of first zone in arg node's generic zonelist. 6324 * Used for initializing percpu 'numa_mem', which is used primarily 6325 * for kernel allocations, so use GFP_KERNEL flags to locate zonelist. 6326 */ 6327 int local_memory_node(int node) 6328 { 6329 struct zoneref *z; 6330 6331 z = first_zones_zonelist(node_zonelist(node, GFP_KERNEL), 6332 gfp_zone(GFP_KERNEL), 6333 NULL); 6334 return zone_to_nid(z->zone); 6335 } 6336 #endif 6337 6338 static void setup_min_unmapped_ratio(void); 6339 static void setup_min_slab_ratio(void); 6340 #else /* CONFIG_NUMA */ 6341 6342 static void build_zonelists(pg_data_t *pgdat) 6343 { 6344 int node, local_node; 6345 struct zoneref *zonerefs; 6346 int nr_zones; 6347 6348 local_node = pgdat->node_id; 6349 6350 zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs; 6351 nr_zones = build_zonerefs_node(pgdat, zonerefs); 6352 zonerefs += nr_zones; 6353 6354 /* 6355 * Now we build the zonelist so that it contains the zones 6356 * of all the other nodes. 6357 * We don't want to pressure a particular node, so when 6358 * building the zones for node N, we make sure that the 6359 * zones coming right after the local ones are those from 6360 * node N+1 (modulo N) 6361 */ 6362 for (node = local_node + 1; node < MAX_NUMNODES; node++) { 6363 if (!node_online(node)) 6364 continue; 6365 nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs); 6366 zonerefs += nr_zones; 6367 } 6368 for (node = 0; node < local_node; node++) { 6369 if (!node_online(node)) 6370 continue; 6371 nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs); 6372 zonerefs += nr_zones; 6373 } 6374 6375 zonerefs->zone = NULL; 6376 zonerefs->zone_idx = 0; 6377 } 6378 6379 #endif /* CONFIG_NUMA */ 6380 6381 /* 6382 * Boot pageset table. One per cpu which is going to be used for all 6383 * zones and all nodes. The parameters will be set in such a way 6384 * that an item put on a list will immediately be handed over to 6385 * the buddy list. This is safe since pageset manipulation is done 6386 * with interrupts disabled. 6387 * 6388 * The boot_pagesets must be kept even after bootup is complete for 6389 * unused processors and/or zones. They do play a role for bootstrapping 6390 * hotplugged processors. 6391 * 6392 * zoneinfo_show() and maybe other functions do 6393 * not check if the processor is online before following the pageset pointer. 6394 * Other parts of the kernel may not check if the zone is available. 6395 */ 6396 static void per_cpu_pages_init(struct per_cpu_pages *pcp, struct per_cpu_zonestat *pzstats); 6397 /* These effectively disable the pcplists in the boot pageset completely */ 6398 #define BOOT_PAGESET_HIGH 0 6399 #define BOOT_PAGESET_BATCH 1 6400 static DEFINE_PER_CPU(struct per_cpu_pages, boot_pageset); 6401 static DEFINE_PER_CPU(struct per_cpu_zonestat, boot_zonestats); 6402 DEFINE_PER_CPU(struct per_cpu_nodestat, boot_nodestats); 6403 6404 static void __build_all_zonelists(void *data) 6405 { 6406 int nid; 6407 int __maybe_unused cpu; 6408 pg_data_t *self = data; 6409 static DEFINE_SPINLOCK(lock); 6410 6411 spin_lock(&lock); 6412 6413 #ifdef CONFIG_NUMA 6414 memset(node_load, 0, sizeof(node_load)); 6415 #endif 6416 6417 /* 6418 * This node is hotadded and no memory is yet present. So just 6419 * building zonelists is fine - no need to touch other nodes. 6420 */ 6421 if (self && !node_online(self->node_id)) { 6422 build_zonelists(self); 6423 } else { 6424 /* 6425 * All possible nodes have pgdat preallocated 6426 * in free_area_init 6427 */ 6428 for_each_node(nid) { 6429 pg_data_t *pgdat = NODE_DATA(nid); 6430 6431 build_zonelists(pgdat); 6432 } 6433 6434 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 6435 /* 6436 * We now know the "local memory node" for each node-- 6437 * i.e., the node of the first zone in the generic zonelist. 6438 * Set up numa_mem percpu variable for on-line cpus. During 6439 * boot, only the boot cpu should be on-line; we'll init the 6440 * secondary cpus' numa_mem as they come on-line. During 6441 * node/memory hotplug, we'll fixup all on-line cpus. 6442 */ 6443 for_each_online_cpu(cpu) 6444 set_cpu_numa_mem(cpu, local_memory_node(cpu_to_node(cpu))); 6445 #endif 6446 } 6447 6448 spin_unlock(&lock); 6449 } 6450 6451 static noinline void __init 6452 build_all_zonelists_init(void) 6453 { 6454 int cpu; 6455 6456 __build_all_zonelists(NULL); 6457 6458 /* 6459 * Initialize the boot_pagesets that are going to be used 6460 * for bootstrapping processors. The real pagesets for 6461 * each zone will be allocated later when the per cpu 6462 * allocator is available. 6463 * 6464 * boot_pagesets are used also for bootstrapping offline 6465 * cpus if the system is already booted because the pagesets 6466 * are needed to initialize allocators on a specific cpu too. 6467 * F.e. the percpu allocator needs the page allocator which 6468 * needs the percpu allocator in order to allocate its pagesets 6469 * (a chicken-egg dilemma). 6470 */ 6471 for_each_possible_cpu(cpu) 6472 per_cpu_pages_init(&per_cpu(boot_pageset, cpu), &per_cpu(boot_zonestats, cpu)); 6473 6474 mminit_verify_zonelist(); 6475 cpuset_init_current_mems_allowed(); 6476 } 6477 6478 /* 6479 * unless system_state == SYSTEM_BOOTING. 6480 * 6481 * __ref due to call of __init annotated helper build_all_zonelists_init 6482 * [protected by SYSTEM_BOOTING]. 6483 */ 6484 void __ref build_all_zonelists(pg_data_t *pgdat) 6485 { 6486 unsigned long vm_total_pages; 6487 6488 if (system_state == SYSTEM_BOOTING) { 6489 build_all_zonelists_init(); 6490 } else { 6491 __build_all_zonelists(pgdat); 6492 /* cpuset refresh routine should be here */ 6493 } 6494 /* Get the number of free pages beyond high watermark in all zones. */ 6495 vm_total_pages = nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE)); 6496 /* 6497 * Disable grouping by mobility if the number of pages in the 6498 * system is too low to allow the mechanism to work. It would be 6499 * more accurate, but expensive to check per-zone. This check is 6500 * made on memory-hotadd so a system can start with mobility 6501 * disabled and enable it later 6502 */ 6503 if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES)) 6504 page_group_by_mobility_disabled = 1; 6505 else 6506 page_group_by_mobility_disabled = 0; 6507 6508 pr_info("Built %u zonelists, mobility grouping %s. Total pages: %ld\n", 6509 nr_online_nodes, 6510 page_group_by_mobility_disabled ? "off" : "on", 6511 vm_total_pages); 6512 #ifdef CONFIG_NUMA 6513 pr_info("Policy zone: %s\n", zone_names[policy_zone]); 6514 #endif 6515 } 6516 6517 /* If zone is ZONE_MOVABLE but memory is mirrored, it is an overlapped init */ 6518 static bool __meminit 6519 overlap_memmap_init(unsigned long zone, unsigned long *pfn) 6520 { 6521 static struct memblock_region *r; 6522 6523 if (mirrored_kernelcore && zone == ZONE_MOVABLE) { 6524 if (!r || *pfn >= memblock_region_memory_end_pfn(r)) { 6525 for_each_mem_region(r) { 6526 if (*pfn < memblock_region_memory_end_pfn(r)) 6527 break; 6528 } 6529 } 6530 if (*pfn >= memblock_region_memory_base_pfn(r) && 6531 memblock_is_mirror(r)) { 6532 *pfn = memblock_region_memory_end_pfn(r); 6533 return true; 6534 } 6535 } 6536 return false; 6537 } 6538 6539 /* 6540 * Initially all pages are reserved - free ones are freed 6541 * up by memblock_free_all() once the early boot process is 6542 * done. Non-atomic initialization, single-pass. 6543 * 6544 * All aligned pageblocks are initialized to the specified migratetype 6545 * (usually MIGRATE_MOVABLE). Besides setting the migratetype, no related 6546 * zone stats (e.g., nr_isolate_pageblock) are touched. 6547 */ 6548 void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone, 6549 unsigned long start_pfn, unsigned long zone_end_pfn, 6550 enum meminit_context context, 6551 struct vmem_altmap *altmap, int migratetype) 6552 { 6553 unsigned long pfn, end_pfn = start_pfn + size; 6554 struct page *page; 6555 6556 if (highest_memmap_pfn < end_pfn - 1) 6557 highest_memmap_pfn = end_pfn - 1; 6558 6559 #ifdef CONFIG_ZONE_DEVICE 6560 /* 6561 * Honor reservation requested by the driver for this ZONE_DEVICE 6562 * memory. We limit the total number of pages to initialize to just 6563 * those that might contain the memory mapping. We will defer the 6564 * ZONE_DEVICE page initialization until after we have released 6565 * the hotplug lock. 6566 */ 6567 if (zone == ZONE_DEVICE) { 6568 if (!altmap) 6569 return; 6570 6571 if (start_pfn == altmap->base_pfn) 6572 start_pfn += altmap->reserve; 6573 end_pfn = altmap->base_pfn + vmem_altmap_offset(altmap); 6574 } 6575 #endif 6576 6577 for (pfn = start_pfn; pfn < end_pfn; ) { 6578 /* 6579 * There can be holes in boot-time mem_map[]s handed to this 6580 * function. They do not exist on hotplugged memory. 6581 */ 6582 if (context == MEMINIT_EARLY) { 6583 if (overlap_memmap_init(zone, &pfn)) 6584 continue; 6585 if (defer_init(nid, pfn, zone_end_pfn)) 6586 break; 6587 } 6588 6589 page = pfn_to_page(pfn); 6590 __init_single_page(page, pfn, zone, nid); 6591 if (context == MEMINIT_HOTPLUG) 6592 __SetPageReserved(page); 6593 6594 /* 6595 * Usually, we want to mark the pageblock MIGRATE_MOVABLE, 6596 * such that unmovable allocations won't be scattered all 6597 * over the place during system boot. 6598 */ 6599 if (IS_ALIGNED(pfn, pageblock_nr_pages)) { 6600 set_pageblock_migratetype(page, migratetype); 6601 cond_resched(); 6602 } 6603 pfn++; 6604 } 6605 } 6606 6607 #ifdef CONFIG_ZONE_DEVICE 6608 static void __ref __init_zone_device_page(struct page *page, unsigned long pfn, 6609 unsigned long zone_idx, int nid, 6610 struct dev_pagemap *pgmap) 6611 { 6612 6613 __init_single_page(page, pfn, zone_idx, nid); 6614 6615 /* 6616 * Mark page reserved as it will need to wait for onlining 6617 * phase for it to be fully associated with a zone. 6618 * 6619 * We can use the non-atomic __set_bit operation for setting 6620 * the flag as we are still initializing the pages. 6621 */ 6622 __SetPageReserved(page); 6623 6624 /* 6625 * ZONE_DEVICE pages union ->lru with a ->pgmap back pointer 6626 * and zone_device_data. It is a bug if a ZONE_DEVICE page is 6627 * ever freed or placed on a driver-private list. 6628 */ 6629 page->pgmap = pgmap; 6630 page->zone_device_data = NULL; 6631 6632 /* 6633 * Mark the block movable so that blocks are reserved for 6634 * movable at startup. This will force kernel allocations 6635 * to reserve their blocks rather than leaking throughout 6636 * the address space during boot when many long-lived 6637 * kernel allocations are made. 6638 * 6639 * Please note that MEMINIT_HOTPLUG path doesn't clear memmap 6640 * because this is done early in section_activate() 6641 */ 6642 if (IS_ALIGNED(pfn, pageblock_nr_pages)) { 6643 set_pageblock_migratetype(page, MIGRATE_MOVABLE); 6644 cond_resched(); 6645 } 6646 } 6647 6648 static void __ref memmap_init_compound(struct page *head, 6649 unsigned long head_pfn, 6650 unsigned long zone_idx, int nid, 6651 struct dev_pagemap *pgmap, 6652 unsigned long nr_pages) 6653 { 6654 unsigned long pfn, end_pfn = head_pfn + nr_pages; 6655 unsigned int order = pgmap->vmemmap_shift; 6656 6657 __SetPageHead(head); 6658 for (pfn = head_pfn + 1; pfn < end_pfn; pfn++) { 6659 struct page *page = pfn_to_page(pfn); 6660 6661 __init_zone_device_page(page, pfn, zone_idx, nid, pgmap); 6662 prep_compound_tail(head, pfn - head_pfn); 6663 set_page_count(page, 0); 6664 6665 /* 6666 * The first tail page stores compound_mapcount_ptr() and 6667 * compound_order() and the second tail page stores 6668 * compound_pincount_ptr(). Call prep_compound_head() after 6669 * the first and second tail pages have been initialized to 6670 * not have the data overwritten. 6671 */ 6672 if (pfn == head_pfn + 2) 6673 prep_compound_head(head, order); 6674 } 6675 } 6676 6677 void __ref memmap_init_zone_device(struct zone *zone, 6678 unsigned long start_pfn, 6679 unsigned long nr_pages, 6680 struct dev_pagemap *pgmap) 6681 { 6682 unsigned long pfn, end_pfn = start_pfn + nr_pages; 6683 struct pglist_data *pgdat = zone->zone_pgdat; 6684 struct vmem_altmap *altmap = pgmap_altmap(pgmap); 6685 unsigned int pfns_per_compound = pgmap_vmemmap_nr(pgmap); 6686 unsigned long zone_idx = zone_idx(zone); 6687 unsigned long start = jiffies; 6688 int nid = pgdat->node_id; 6689 6690 if (WARN_ON_ONCE(!pgmap || zone_idx(zone) != ZONE_DEVICE)) 6691 return; 6692 6693 /* 6694 * The call to memmap_init should have already taken care 6695 * of the pages reserved for the memmap, so we can just jump to 6696 * the end of that region and start processing the device pages. 6697 */ 6698 if (altmap) { 6699 start_pfn = altmap->base_pfn + vmem_altmap_offset(altmap); 6700 nr_pages = end_pfn - start_pfn; 6701 } 6702 6703 for (pfn = start_pfn; pfn < end_pfn; pfn += pfns_per_compound) { 6704 struct page *page = pfn_to_page(pfn); 6705 6706 __init_zone_device_page(page, pfn, zone_idx, nid, pgmap); 6707 6708 if (pfns_per_compound == 1) 6709 continue; 6710 6711 memmap_init_compound(page, pfn, zone_idx, nid, pgmap, 6712 pfns_per_compound); 6713 } 6714 6715 pr_info("%s initialised %lu pages in %ums\n", __func__, 6716 nr_pages, jiffies_to_msecs(jiffies - start)); 6717 } 6718 6719 #endif 6720 static void __meminit zone_init_free_lists(struct zone *zone) 6721 { 6722 unsigned int order, t; 6723 for_each_migratetype_order(order, t) { 6724 INIT_LIST_HEAD(&zone->free_area[order].free_list[t]); 6725 zone->free_area[order].nr_free = 0; 6726 } 6727 } 6728 6729 /* 6730 * Only struct pages that correspond to ranges defined by memblock.memory 6731 * are zeroed and initialized by going through __init_single_page() during 6732 * memmap_init_zone_range(). 6733 * 6734 * But, there could be struct pages that correspond to holes in 6735 * memblock.memory. This can happen because of the following reasons: 6736 * - physical memory bank size is not necessarily the exact multiple of the 6737 * arbitrary section size 6738 * - early reserved memory may not be listed in memblock.memory 6739 * - memory layouts defined with memmap= kernel parameter may not align 6740 * nicely with memmap sections 6741 * 6742 * Explicitly initialize those struct pages so that: 6743 * - PG_Reserved is set 6744 * - zone and node links point to zone and node that span the page if the 6745 * hole is in the middle of a zone 6746 * - zone and node links point to adjacent zone/node if the hole falls on 6747 * the zone boundary; the pages in such holes will be prepended to the 6748 * zone/node above the hole except for the trailing pages in the last 6749 * section that will be appended to the zone/node below. 6750 */ 6751 static void __init init_unavailable_range(unsigned long spfn, 6752 unsigned long epfn, 6753 int zone, int node) 6754 { 6755 unsigned long pfn; 6756 u64 pgcnt = 0; 6757 6758 for (pfn = spfn; pfn < epfn; pfn++) { 6759 if (!pfn_valid(ALIGN_DOWN(pfn, pageblock_nr_pages))) { 6760 pfn = ALIGN_DOWN(pfn, pageblock_nr_pages) 6761 + pageblock_nr_pages - 1; 6762 continue; 6763 } 6764 __init_single_page(pfn_to_page(pfn), pfn, zone, node); 6765 __SetPageReserved(pfn_to_page(pfn)); 6766 pgcnt++; 6767 } 6768 6769 if (pgcnt) 6770 pr_info("On node %d, zone %s: %lld pages in unavailable ranges", 6771 node, zone_names[zone], pgcnt); 6772 } 6773 6774 static void __init memmap_init_zone_range(struct zone *zone, 6775 unsigned long start_pfn, 6776 unsigned long end_pfn, 6777 unsigned long *hole_pfn) 6778 { 6779 unsigned long zone_start_pfn = zone->zone_start_pfn; 6780 unsigned long zone_end_pfn = zone_start_pfn + zone->spanned_pages; 6781 int nid = zone_to_nid(zone), zone_id = zone_idx(zone); 6782 6783 start_pfn = clamp(start_pfn, zone_start_pfn, zone_end_pfn); 6784 end_pfn = clamp(end_pfn, zone_start_pfn, zone_end_pfn); 6785 6786 if (start_pfn >= end_pfn) 6787 return; 6788 6789 memmap_init_range(end_pfn - start_pfn, nid, zone_id, start_pfn, 6790 zone_end_pfn, MEMINIT_EARLY, NULL, MIGRATE_MOVABLE); 6791 6792 if (*hole_pfn < start_pfn) 6793 init_unavailable_range(*hole_pfn, start_pfn, zone_id, nid); 6794 6795 *hole_pfn = end_pfn; 6796 } 6797 6798 static void __init memmap_init(void) 6799 { 6800 unsigned long start_pfn, end_pfn; 6801 unsigned long hole_pfn = 0; 6802 int i, j, zone_id = 0, nid; 6803 6804 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 6805 struct pglist_data *node = NODE_DATA(nid); 6806 6807 for (j = 0; j < MAX_NR_ZONES; j++) { 6808 struct zone *zone = node->node_zones + j; 6809 6810 if (!populated_zone(zone)) 6811 continue; 6812 6813 memmap_init_zone_range(zone, start_pfn, end_pfn, 6814 &hole_pfn); 6815 zone_id = j; 6816 } 6817 } 6818 6819 #ifdef CONFIG_SPARSEMEM 6820 /* 6821 * Initialize the memory map for hole in the range [memory_end, 6822 * section_end]. 6823 * Append the pages in this hole to the highest zone in the last 6824 * node. 6825 * The call to init_unavailable_range() is outside the ifdef to 6826 * silence the compiler warining about zone_id set but not used; 6827 * for FLATMEM it is a nop anyway 6828 */ 6829 end_pfn = round_up(end_pfn, PAGES_PER_SECTION); 6830 if (hole_pfn < end_pfn) 6831 #endif 6832 init_unavailable_range(hole_pfn, end_pfn, zone_id, nid); 6833 } 6834 6835 void __init *memmap_alloc(phys_addr_t size, phys_addr_t align, 6836 phys_addr_t min_addr, int nid, bool exact_nid) 6837 { 6838 void *ptr; 6839 6840 if (exact_nid) 6841 ptr = memblock_alloc_exact_nid_raw(size, align, min_addr, 6842 MEMBLOCK_ALLOC_ACCESSIBLE, 6843 nid); 6844 else 6845 ptr = memblock_alloc_try_nid_raw(size, align, min_addr, 6846 MEMBLOCK_ALLOC_ACCESSIBLE, 6847 nid); 6848 6849 if (ptr && size > 0) 6850 page_init_poison(ptr, size); 6851 6852 return ptr; 6853 } 6854 6855 static int zone_batchsize(struct zone *zone) 6856 { 6857 #ifdef CONFIG_MMU 6858 int batch; 6859 6860 /* 6861 * The number of pages to batch allocate is either ~0.1% 6862 * of the zone or 1MB, whichever is smaller. The batch 6863 * size is striking a balance between allocation latency 6864 * and zone lock contention. 6865 */ 6866 batch = min(zone_managed_pages(zone) >> 10, (1024 * 1024) / PAGE_SIZE); 6867 batch /= 4; /* We effectively *= 4 below */ 6868 if (batch < 1) 6869 batch = 1; 6870 6871 /* 6872 * Clamp the batch to a 2^n - 1 value. Having a power 6873 * of 2 value was found to be more likely to have 6874 * suboptimal cache aliasing properties in some cases. 6875 * 6876 * For example if 2 tasks are alternately allocating 6877 * batches of pages, one task can end up with a lot 6878 * of pages of one half of the possible page colors 6879 * and the other with pages of the other colors. 6880 */ 6881 batch = rounddown_pow_of_two(batch + batch/2) - 1; 6882 6883 return batch; 6884 6885 #else 6886 /* The deferral and batching of frees should be suppressed under NOMMU 6887 * conditions. 6888 * 6889 * The problem is that NOMMU needs to be able to allocate large chunks 6890 * of contiguous memory as there's no hardware page translation to 6891 * assemble apparent contiguous memory from discontiguous pages. 6892 * 6893 * Queueing large contiguous runs of pages for batching, however, 6894 * causes the pages to actually be freed in smaller chunks. As there 6895 * can be a significant delay between the individual batches being 6896 * recycled, this leads to the once large chunks of space being 6897 * fragmented and becoming unavailable for high-order allocations. 6898 */ 6899 return 0; 6900 #endif 6901 } 6902 6903 static int zone_highsize(struct zone *zone, int batch, int cpu_online) 6904 { 6905 #ifdef CONFIG_MMU 6906 int high; 6907 int nr_split_cpus; 6908 unsigned long total_pages; 6909 6910 if (!percpu_pagelist_high_fraction) { 6911 /* 6912 * By default, the high value of the pcp is based on the zone 6913 * low watermark so that if they are full then background 6914 * reclaim will not be started prematurely. 6915 */ 6916 total_pages = low_wmark_pages(zone); 6917 } else { 6918 /* 6919 * If percpu_pagelist_high_fraction is configured, the high 6920 * value is based on a fraction of the managed pages in the 6921 * zone. 6922 */ 6923 total_pages = zone_managed_pages(zone) / percpu_pagelist_high_fraction; 6924 } 6925 6926 /* 6927 * Split the high value across all online CPUs local to the zone. Note 6928 * that early in boot that CPUs may not be online yet and that during 6929 * CPU hotplug that the cpumask is not yet updated when a CPU is being 6930 * onlined. For memory nodes that have no CPUs, split pcp->high across 6931 * all online CPUs to mitigate the risk that reclaim is triggered 6932 * prematurely due to pages stored on pcp lists. 6933 */ 6934 nr_split_cpus = cpumask_weight(cpumask_of_node(zone_to_nid(zone))) + cpu_online; 6935 if (!nr_split_cpus) 6936 nr_split_cpus = num_online_cpus(); 6937 high = total_pages / nr_split_cpus; 6938 6939 /* 6940 * Ensure high is at least batch*4. The multiple is based on the 6941 * historical relationship between high and batch. 6942 */ 6943 high = max(high, batch << 2); 6944 6945 return high; 6946 #else 6947 return 0; 6948 #endif 6949 } 6950 6951 /* 6952 * pcp->high and pcp->batch values are related and generally batch is lower 6953 * than high. They are also related to pcp->count such that count is lower 6954 * than high, and as soon as it reaches high, the pcplist is flushed. 6955 * 6956 * However, guaranteeing these relations at all times would require e.g. write 6957 * barriers here but also careful usage of read barriers at the read side, and 6958 * thus be prone to error and bad for performance. Thus the update only prevents 6959 * store tearing. Any new users of pcp->batch and pcp->high should ensure they 6960 * can cope with those fields changing asynchronously, and fully trust only the 6961 * pcp->count field on the local CPU with interrupts disabled. 6962 * 6963 * mutex_is_locked(&pcp_batch_high_lock) required when calling this function 6964 * outside of boot time (or some other assurance that no concurrent updaters 6965 * exist). 6966 */ 6967 static void pageset_update(struct per_cpu_pages *pcp, unsigned long high, 6968 unsigned long batch) 6969 { 6970 WRITE_ONCE(pcp->batch, batch); 6971 WRITE_ONCE(pcp->high, high); 6972 } 6973 6974 static void per_cpu_pages_init(struct per_cpu_pages *pcp, struct per_cpu_zonestat *pzstats) 6975 { 6976 int pindex; 6977 6978 memset(pcp, 0, sizeof(*pcp)); 6979 memset(pzstats, 0, sizeof(*pzstats)); 6980 6981 for (pindex = 0; pindex < NR_PCP_LISTS; pindex++) 6982 INIT_LIST_HEAD(&pcp->lists[pindex]); 6983 6984 /* 6985 * Set batch and high values safe for a boot pageset. A true percpu 6986 * pageset's initialization will update them subsequently. Here we don't 6987 * need to be as careful as pageset_update() as nobody can access the 6988 * pageset yet. 6989 */ 6990 pcp->high = BOOT_PAGESET_HIGH; 6991 pcp->batch = BOOT_PAGESET_BATCH; 6992 pcp->free_factor = 0; 6993 } 6994 6995 static void __zone_set_pageset_high_and_batch(struct zone *zone, unsigned long high, 6996 unsigned long batch) 6997 { 6998 struct per_cpu_pages *pcp; 6999 int cpu; 7000 7001 for_each_possible_cpu(cpu) { 7002 pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); 7003 pageset_update(pcp, high, batch); 7004 } 7005 } 7006 7007 /* 7008 * Calculate and set new high and batch values for all per-cpu pagesets of a 7009 * zone based on the zone's size. 7010 */ 7011 static void zone_set_pageset_high_and_batch(struct zone *zone, int cpu_online) 7012 { 7013 int new_high, new_batch; 7014 7015 new_batch = max(1, zone_batchsize(zone)); 7016 new_high = zone_highsize(zone, new_batch, cpu_online); 7017 7018 if (zone->pageset_high == new_high && 7019 zone->pageset_batch == new_batch) 7020 return; 7021 7022 zone->pageset_high = new_high; 7023 zone->pageset_batch = new_batch; 7024 7025 __zone_set_pageset_high_and_batch(zone, new_high, new_batch); 7026 } 7027 7028 void __meminit setup_zone_pageset(struct zone *zone) 7029 { 7030 int cpu; 7031 7032 /* Size may be 0 on !SMP && !NUMA */ 7033 if (sizeof(struct per_cpu_zonestat) > 0) 7034 zone->per_cpu_zonestats = alloc_percpu(struct per_cpu_zonestat); 7035 7036 zone->per_cpu_pageset = alloc_percpu(struct per_cpu_pages); 7037 for_each_possible_cpu(cpu) { 7038 struct per_cpu_pages *pcp; 7039 struct per_cpu_zonestat *pzstats; 7040 7041 pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); 7042 pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu); 7043 per_cpu_pages_init(pcp, pzstats); 7044 } 7045 7046 zone_set_pageset_high_and_batch(zone, 0); 7047 } 7048 7049 /* 7050 * Allocate per cpu pagesets and initialize them. 7051 * Before this call only boot pagesets were available. 7052 */ 7053 void __init setup_per_cpu_pageset(void) 7054 { 7055 struct pglist_data *pgdat; 7056 struct zone *zone; 7057 int __maybe_unused cpu; 7058 7059 for_each_populated_zone(zone) 7060 setup_zone_pageset(zone); 7061 7062 #ifdef CONFIG_NUMA 7063 /* 7064 * Unpopulated zones continue using the boot pagesets. 7065 * The numa stats for these pagesets need to be reset. 7066 * Otherwise, they will end up skewing the stats of 7067 * the nodes these zones are associated with. 7068 */ 7069 for_each_possible_cpu(cpu) { 7070 struct per_cpu_zonestat *pzstats = &per_cpu(boot_zonestats, cpu); 7071 memset(pzstats->vm_numa_event, 0, 7072 sizeof(pzstats->vm_numa_event)); 7073 } 7074 #endif 7075 7076 for_each_online_pgdat(pgdat) 7077 pgdat->per_cpu_nodestats = 7078 alloc_percpu(struct per_cpu_nodestat); 7079 } 7080 7081 static __meminit void zone_pcp_init(struct zone *zone) 7082 { 7083 /* 7084 * per cpu subsystem is not up at this point. The following code 7085 * relies on the ability of the linker to provide the 7086 * offset of a (static) per cpu variable into the per cpu area. 7087 */ 7088 zone->per_cpu_pageset = &boot_pageset; 7089 zone->per_cpu_zonestats = &boot_zonestats; 7090 zone->pageset_high = BOOT_PAGESET_HIGH; 7091 zone->pageset_batch = BOOT_PAGESET_BATCH; 7092 7093 if (populated_zone(zone)) 7094 pr_debug(" %s zone: %lu pages, LIFO batch:%u\n", zone->name, 7095 zone->present_pages, zone_batchsize(zone)); 7096 } 7097 7098 void __meminit init_currently_empty_zone(struct zone *zone, 7099 unsigned long zone_start_pfn, 7100 unsigned long size) 7101 { 7102 struct pglist_data *pgdat = zone->zone_pgdat; 7103 int zone_idx = zone_idx(zone) + 1; 7104 7105 if (zone_idx > pgdat->nr_zones) 7106 pgdat->nr_zones = zone_idx; 7107 7108 zone->zone_start_pfn = zone_start_pfn; 7109 7110 mminit_dprintk(MMINIT_TRACE, "memmap_init", 7111 "Initialising map node %d zone %lu pfns %lu -> %lu\n", 7112 pgdat->node_id, 7113 (unsigned long)zone_idx(zone), 7114 zone_start_pfn, (zone_start_pfn + size)); 7115 7116 zone_init_free_lists(zone); 7117 zone->initialized = 1; 7118 } 7119 7120 /** 7121 * get_pfn_range_for_nid - Return the start and end page frames for a node 7122 * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned. 7123 * @start_pfn: Passed by reference. On return, it will have the node start_pfn. 7124 * @end_pfn: Passed by reference. On return, it will have the node end_pfn. 7125 * 7126 * It returns the start and end page frame of a node based on information 7127 * provided by memblock_set_node(). If called for a node 7128 * with no available memory, a warning is printed and the start and end 7129 * PFNs will be 0. 7130 */ 7131 void __init get_pfn_range_for_nid(unsigned int nid, 7132 unsigned long *start_pfn, unsigned long *end_pfn) 7133 { 7134 unsigned long this_start_pfn, this_end_pfn; 7135 int i; 7136 7137 *start_pfn = -1UL; 7138 *end_pfn = 0; 7139 7140 for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) { 7141 *start_pfn = min(*start_pfn, this_start_pfn); 7142 *end_pfn = max(*end_pfn, this_end_pfn); 7143 } 7144 7145 if (*start_pfn == -1UL) 7146 *start_pfn = 0; 7147 } 7148 7149 /* 7150 * This finds a zone that can be used for ZONE_MOVABLE pages. The 7151 * assumption is made that zones within a node are ordered in monotonic 7152 * increasing memory addresses so that the "highest" populated zone is used 7153 */ 7154 static void __init find_usable_zone_for_movable(void) 7155 { 7156 int zone_index; 7157 for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) { 7158 if (zone_index == ZONE_MOVABLE) 7159 continue; 7160 7161 if (arch_zone_highest_possible_pfn[zone_index] > 7162 arch_zone_lowest_possible_pfn[zone_index]) 7163 break; 7164 } 7165 7166 VM_BUG_ON(zone_index == -1); 7167 movable_zone = zone_index; 7168 } 7169 7170 /* 7171 * The zone ranges provided by the architecture do not include ZONE_MOVABLE 7172 * because it is sized independent of architecture. Unlike the other zones, 7173 * the starting point for ZONE_MOVABLE is not fixed. It may be different 7174 * in each node depending on the size of each node and how evenly kernelcore 7175 * is distributed. This helper function adjusts the zone ranges 7176 * provided by the architecture for a given node by using the end of the 7177 * highest usable zone for ZONE_MOVABLE. This preserves the assumption that 7178 * zones within a node are in order of monotonic increases memory addresses 7179 */ 7180 static void __init adjust_zone_range_for_zone_movable(int nid, 7181 unsigned long zone_type, 7182 unsigned long node_start_pfn, 7183 unsigned long node_end_pfn, 7184 unsigned long *zone_start_pfn, 7185 unsigned long *zone_end_pfn) 7186 { 7187 /* Only adjust if ZONE_MOVABLE is on this node */ 7188 if (zone_movable_pfn[nid]) { 7189 /* Size ZONE_MOVABLE */ 7190 if (zone_type == ZONE_MOVABLE) { 7191 *zone_start_pfn = zone_movable_pfn[nid]; 7192 *zone_end_pfn = min(node_end_pfn, 7193 arch_zone_highest_possible_pfn[movable_zone]); 7194 7195 /* Adjust for ZONE_MOVABLE starting within this range */ 7196 } else if (!mirrored_kernelcore && 7197 *zone_start_pfn < zone_movable_pfn[nid] && 7198 *zone_end_pfn > zone_movable_pfn[nid]) { 7199 *zone_end_pfn = zone_movable_pfn[nid]; 7200 7201 /* Check if this whole range is within ZONE_MOVABLE */ 7202 } else if (*zone_start_pfn >= zone_movable_pfn[nid]) 7203 *zone_start_pfn = *zone_end_pfn; 7204 } 7205 } 7206 7207 /* 7208 * Return the number of pages a zone spans in a node, including holes 7209 * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node() 7210 */ 7211 static unsigned long __init zone_spanned_pages_in_node(int nid, 7212 unsigned long zone_type, 7213 unsigned long node_start_pfn, 7214 unsigned long node_end_pfn, 7215 unsigned long *zone_start_pfn, 7216 unsigned long *zone_end_pfn) 7217 { 7218 unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type]; 7219 unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type]; 7220 /* When hotadd a new node from cpu_up(), the node should be empty */ 7221 if (!node_start_pfn && !node_end_pfn) 7222 return 0; 7223 7224 /* Get the start and end of the zone */ 7225 *zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high); 7226 *zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high); 7227 adjust_zone_range_for_zone_movable(nid, zone_type, 7228 node_start_pfn, node_end_pfn, 7229 zone_start_pfn, zone_end_pfn); 7230 7231 /* Check that this node has pages within the zone's required range */ 7232 if (*zone_end_pfn < node_start_pfn || *zone_start_pfn > node_end_pfn) 7233 return 0; 7234 7235 /* Move the zone boundaries inside the node if necessary */ 7236 *zone_end_pfn = min(*zone_end_pfn, node_end_pfn); 7237 *zone_start_pfn = max(*zone_start_pfn, node_start_pfn); 7238 7239 /* Return the spanned pages */ 7240 return *zone_end_pfn - *zone_start_pfn; 7241 } 7242 7243 /* 7244 * Return the number of holes in a range on a node. If nid is MAX_NUMNODES, 7245 * then all holes in the requested range will be accounted for. 7246 */ 7247 unsigned long __init __absent_pages_in_range(int nid, 7248 unsigned long range_start_pfn, 7249 unsigned long range_end_pfn) 7250 { 7251 unsigned long nr_absent = range_end_pfn - range_start_pfn; 7252 unsigned long start_pfn, end_pfn; 7253 int i; 7254 7255 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { 7256 start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn); 7257 end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn); 7258 nr_absent -= end_pfn - start_pfn; 7259 } 7260 return nr_absent; 7261 } 7262 7263 /** 7264 * absent_pages_in_range - Return number of page frames in holes within a range 7265 * @start_pfn: The start PFN to start searching for holes 7266 * @end_pfn: The end PFN to stop searching for holes 7267 * 7268 * Return: the number of pages frames in memory holes within a range. 7269 */ 7270 unsigned long __init absent_pages_in_range(unsigned long start_pfn, 7271 unsigned long end_pfn) 7272 { 7273 return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn); 7274 } 7275 7276 /* Return the number of page frames in holes in a zone on a node */ 7277 static unsigned long __init zone_absent_pages_in_node(int nid, 7278 unsigned long zone_type, 7279 unsigned long node_start_pfn, 7280 unsigned long node_end_pfn) 7281 { 7282 unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type]; 7283 unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type]; 7284 unsigned long zone_start_pfn, zone_end_pfn; 7285 unsigned long nr_absent; 7286 7287 /* When hotadd a new node from cpu_up(), the node should be empty */ 7288 if (!node_start_pfn && !node_end_pfn) 7289 return 0; 7290 7291 zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high); 7292 zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high); 7293 7294 adjust_zone_range_for_zone_movable(nid, zone_type, 7295 node_start_pfn, node_end_pfn, 7296 &zone_start_pfn, &zone_end_pfn); 7297 nr_absent = __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn); 7298 7299 /* 7300 * ZONE_MOVABLE handling. 7301 * Treat pages to be ZONE_MOVABLE in ZONE_NORMAL as absent pages 7302 * and vice versa. 7303 */ 7304 if (mirrored_kernelcore && zone_movable_pfn[nid]) { 7305 unsigned long start_pfn, end_pfn; 7306 struct memblock_region *r; 7307 7308 for_each_mem_region(r) { 7309 start_pfn = clamp(memblock_region_memory_base_pfn(r), 7310 zone_start_pfn, zone_end_pfn); 7311 end_pfn = clamp(memblock_region_memory_end_pfn(r), 7312 zone_start_pfn, zone_end_pfn); 7313 7314 if (zone_type == ZONE_MOVABLE && 7315 memblock_is_mirror(r)) 7316 nr_absent += end_pfn - start_pfn; 7317 7318 if (zone_type == ZONE_NORMAL && 7319 !memblock_is_mirror(r)) 7320 nr_absent += end_pfn - start_pfn; 7321 } 7322 } 7323 7324 return nr_absent; 7325 } 7326 7327 static void __init calculate_node_totalpages(struct pglist_data *pgdat, 7328 unsigned long node_start_pfn, 7329 unsigned long node_end_pfn) 7330 { 7331 unsigned long realtotalpages = 0, totalpages = 0; 7332 enum zone_type i; 7333 7334 for (i = 0; i < MAX_NR_ZONES; i++) { 7335 struct zone *zone = pgdat->node_zones + i; 7336 unsigned long zone_start_pfn, zone_end_pfn; 7337 unsigned long spanned, absent; 7338 unsigned long size, real_size; 7339 7340 spanned = zone_spanned_pages_in_node(pgdat->node_id, i, 7341 node_start_pfn, 7342 node_end_pfn, 7343 &zone_start_pfn, 7344 &zone_end_pfn); 7345 absent = zone_absent_pages_in_node(pgdat->node_id, i, 7346 node_start_pfn, 7347 node_end_pfn); 7348 7349 size = spanned; 7350 real_size = size - absent; 7351 7352 if (size) 7353 zone->zone_start_pfn = zone_start_pfn; 7354 else 7355 zone->zone_start_pfn = 0; 7356 zone->spanned_pages = size; 7357 zone->present_pages = real_size; 7358 #if defined(CONFIG_MEMORY_HOTPLUG) 7359 zone->present_early_pages = real_size; 7360 #endif 7361 7362 totalpages += size; 7363 realtotalpages += real_size; 7364 } 7365 7366 pgdat->node_spanned_pages = totalpages; 7367 pgdat->node_present_pages = realtotalpages; 7368 pr_debug("On node %d totalpages: %lu\n", pgdat->node_id, realtotalpages); 7369 } 7370 7371 #ifndef CONFIG_SPARSEMEM 7372 /* 7373 * Calculate the size of the zone->blockflags rounded to an unsigned long 7374 * Start by making sure zonesize is a multiple of pageblock_order by rounding 7375 * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally 7376 * round what is now in bits to nearest long in bits, then return it in 7377 * bytes. 7378 */ 7379 static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize) 7380 { 7381 unsigned long usemapsize; 7382 7383 zonesize += zone_start_pfn & (pageblock_nr_pages-1); 7384 usemapsize = roundup(zonesize, pageblock_nr_pages); 7385 usemapsize = usemapsize >> pageblock_order; 7386 usemapsize *= NR_PAGEBLOCK_BITS; 7387 usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long)); 7388 7389 return usemapsize / 8; 7390 } 7391 7392 static void __ref setup_usemap(struct zone *zone) 7393 { 7394 unsigned long usemapsize = usemap_size(zone->zone_start_pfn, 7395 zone->spanned_pages); 7396 zone->pageblock_flags = NULL; 7397 if (usemapsize) { 7398 zone->pageblock_flags = 7399 memblock_alloc_node(usemapsize, SMP_CACHE_BYTES, 7400 zone_to_nid(zone)); 7401 if (!zone->pageblock_flags) 7402 panic("Failed to allocate %ld bytes for zone %s pageblock flags on node %d\n", 7403 usemapsize, zone->name, zone_to_nid(zone)); 7404 } 7405 } 7406 #else 7407 static inline void setup_usemap(struct zone *zone) {} 7408 #endif /* CONFIG_SPARSEMEM */ 7409 7410 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE 7411 7412 /* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */ 7413 void __init set_pageblock_order(void) 7414 { 7415 unsigned int order = MAX_ORDER - 1; 7416 7417 /* Check that pageblock_nr_pages has not already been setup */ 7418 if (pageblock_order) 7419 return; 7420 7421 /* Don't let pageblocks exceed the maximum allocation granularity. */ 7422 if (HPAGE_SHIFT > PAGE_SHIFT && HUGETLB_PAGE_ORDER < order) 7423 order = HUGETLB_PAGE_ORDER; 7424 7425 /* 7426 * Assume the largest contiguous order of interest is a huge page. 7427 * This value may be variable depending on boot parameters on IA64 and 7428 * powerpc. 7429 */ 7430 pageblock_order = order; 7431 } 7432 #else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ 7433 7434 /* 7435 * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order() 7436 * is unused as pageblock_order is set at compile-time. See 7437 * include/linux/pageblock-flags.h for the values of pageblock_order based on 7438 * the kernel config 7439 */ 7440 void __init set_pageblock_order(void) 7441 { 7442 } 7443 7444 #endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ 7445 7446 static unsigned long __init calc_memmap_size(unsigned long spanned_pages, 7447 unsigned long present_pages) 7448 { 7449 unsigned long pages = spanned_pages; 7450 7451 /* 7452 * Provide a more accurate estimation if there are holes within 7453 * the zone and SPARSEMEM is in use. If there are holes within the 7454 * zone, each populated memory region may cost us one or two extra 7455 * memmap pages due to alignment because memmap pages for each 7456 * populated regions may not be naturally aligned on page boundary. 7457 * So the (present_pages >> 4) heuristic is a tradeoff for that. 7458 */ 7459 if (spanned_pages > present_pages + (present_pages >> 4) && 7460 IS_ENABLED(CONFIG_SPARSEMEM)) 7461 pages = present_pages; 7462 7463 return PAGE_ALIGN(pages * sizeof(struct page)) >> PAGE_SHIFT; 7464 } 7465 7466 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 7467 static void pgdat_init_split_queue(struct pglist_data *pgdat) 7468 { 7469 struct deferred_split *ds_queue = &pgdat->deferred_split_queue; 7470 7471 spin_lock_init(&ds_queue->split_queue_lock); 7472 INIT_LIST_HEAD(&ds_queue->split_queue); 7473 ds_queue->split_queue_len = 0; 7474 } 7475 #else 7476 static void pgdat_init_split_queue(struct pglist_data *pgdat) {} 7477 #endif 7478 7479 #ifdef CONFIG_COMPACTION 7480 static void pgdat_init_kcompactd(struct pglist_data *pgdat) 7481 { 7482 init_waitqueue_head(&pgdat->kcompactd_wait); 7483 } 7484 #else 7485 static void pgdat_init_kcompactd(struct pglist_data *pgdat) {} 7486 #endif 7487 7488 static void __meminit pgdat_init_internals(struct pglist_data *pgdat) 7489 { 7490 int i; 7491 7492 pgdat_resize_init(pgdat); 7493 7494 pgdat_init_split_queue(pgdat); 7495 pgdat_init_kcompactd(pgdat); 7496 7497 init_waitqueue_head(&pgdat->kswapd_wait); 7498 init_waitqueue_head(&pgdat->pfmemalloc_wait); 7499 7500 for (i = 0; i < NR_VMSCAN_THROTTLE; i++) 7501 init_waitqueue_head(&pgdat->reclaim_wait[i]); 7502 7503 pgdat_page_ext_init(pgdat); 7504 lruvec_init(&pgdat->__lruvec); 7505 } 7506 7507 static void __meminit zone_init_internals(struct zone *zone, enum zone_type idx, int nid, 7508 unsigned long remaining_pages) 7509 { 7510 atomic_long_set(&zone->managed_pages, remaining_pages); 7511 zone_set_nid(zone, nid); 7512 zone->name = zone_names[idx]; 7513 zone->zone_pgdat = NODE_DATA(nid); 7514 spin_lock_init(&zone->lock); 7515 zone_seqlock_init(zone); 7516 zone_pcp_init(zone); 7517 } 7518 7519 /* 7520 * Set up the zone data structures 7521 * - init pgdat internals 7522 * - init all zones belonging to this node 7523 * 7524 * NOTE: this function is only called during memory hotplug 7525 */ 7526 #ifdef CONFIG_MEMORY_HOTPLUG 7527 void __ref free_area_init_core_hotplug(struct pglist_data *pgdat) 7528 { 7529 int nid = pgdat->node_id; 7530 enum zone_type z; 7531 int cpu; 7532 7533 pgdat_init_internals(pgdat); 7534 7535 if (pgdat->per_cpu_nodestats == &boot_nodestats) 7536 pgdat->per_cpu_nodestats = alloc_percpu(struct per_cpu_nodestat); 7537 7538 /* 7539 * Reset the nr_zones, order and highest_zoneidx before reuse. 7540 * Note that kswapd will init kswapd_highest_zoneidx properly 7541 * when it starts in the near future. 7542 */ 7543 pgdat->nr_zones = 0; 7544 pgdat->kswapd_order = 0; 7545 pgdat->kswapd_highest_zoneidx = 0; 7546 pgdat->node_start_pfn = 0; 7547 for_each_online_cpu(cpu) { 7548 struct per_cpu_nodestat *p; 7549 7550 p = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu); 7551 memset(p, 0, sizeof(*p)); 7552 } 7553 7554 for (z = 0; z < MAX_NR_ZONES; z++) 7555 zone_init_internals(&pgdat->node_zones[z], z, nid, 0); 7556 } 7557 #endif 7558 7559 /* 7560 * Set up the zone data structures: 7561 * - mark all pages reserved 7562 * - mark all memory queues empty 7563 * - clear the memory bitmaps 7564 * 7565 * NOTE: pgdat should get zeroed by caller. 7566 * NOTE: this function is only called during early init. 7567 */ 7568 static void __init free_area_init_core(struct pglist_data *pgdat) 7569 { 7570 enum zone_type j; 7571 int nid = pgdat->node_id; 7572 7573 pgdat_init_internals(pgdat); 7574 pgdat->per_cpu_nodestats = &boot_nodestats; 7575 7576 for (j = 0; j < MAX_NR_ZONES; j++) { 7577 struct zone *zone = pgdat->node_zones + j; 7578 unsigned long size, freesize, memmap_pages; 7579 7580 size = zone->spanned_pages; 7581 freesize = zone->present_pages; 7582 7583 /* 7584 * Adjust freesize so that it accounts for how much memory 7585 * is used by this zone for memmap. This affects the watermark 7586 * and per-cpu initialisations 7587 */ 7588 memmap_pages = calc_memmap_size(size, freesize); 7589 if (!is_highmem_idx(j)) { 7590 if (freesize >= memmap_pages) { 7591 freesize -= memmap_pages; 7592 if (memmap_pages) 7593 pr_debug(" %s zone: %lu pages used for memmap\n", 7594 zone_names[j], memmap_pages); 7595 } else 7596 pr_warn(" %s zone: %lu memmap pages exceeds freesize %lu\n", 7597 zone_names[j], memmap_pages, freesize); 7598 } 7599 7600 /* Account for reserved pages */ 7601 if (j == 0 && freesize > dma_reserve) { 7602 freesize -= dma_reserve; 7603 pr_debug(" %s zone: %lu pages reserved\n", zone_names[0], dma_reserve); 7604 } 7605 7606 if (!is_highmem_idx(j)) 7607 nr_kernel_pages += freesize; 7608 /* Charge for highmem memmap if there are enough kernel pages */ 7609 else if (nr_kernel_pages > memmap_pages * 2) 7610 nr_kernel_pages -= memmap_pages; 7611 nr_all_pages += freesize; 7612 7613 /* 7614 * Set an approximate value for lowmem here, it will be adjusted 7615 * when the bootmem allocator frees pages into the buddy system. 7616 * And all highmem pages will be managed by the buddy system. 7617 */ 7618 zone_init_internals(zone, j, nid, freesize); 7619 7620 if (!size) 7621 continue; 7622 7623 set_pageblock_order(); 7624 setup_usemap(zone); 7625 init_currently_empty_zone(zone, zone->zone_start_pfn, size); 7626 } 7627 } 7628 7629 #ifdef CONFIG_FLATMEM 7630 static void __init alloc_node_mem_map(struct pglist_data *pgdat) 7631 { 7632 unsigned long __maybe_unused start = 0; 7633 unsigned long __maybe_unused offset = 0; 7634 7635 /* Skip empty nodes */ 7636 if (!pgdat->node_spanned_pages) 7637 return; 7638 7639 start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1); 7640 offset = pgdat->node_start_pfn - start; 7641 /* ia64 gets its own node_mem_map, before this, without bootmem */ 7642 if (!pgdat->node_mem_map) { 7643 unsigned long size, end; 7644 struct page *map; 7645 7646 /* 7647 * The zone's endpoints aren't required to be MAX_ORDER 7648 * aligned but the node_mem_map endpoints must be in order 7649 * for the buddy allocator to function correctly. 7650 */ 7651 end = pgdat_end_pfn(pgdat); 7652 end = ALIGN(end, MAX_ORDER_NR_PAGES); 7653 size = (end - start) * sizeof(struct page); 7654 map = memmap_alloc(size, SMP_CACHE_BYTES, MEMBLOCK_LOW_LIMIT, 7655 pgdat->node_id, false); 7656 if (!map) 7657 panic("Failed to allocate %ld bytes for node %d memory map\n", 7658 size, pgdat->node_id); 7659 pgdat->node_mem_map = map + offset; 7660 } 7661 pr_debug("%s: node %d, pgdat %08lx, node_mem_map %08lx\n", 7662 __func__, pgdat->node_id, (unsigned long)pgdat, 7663 (unsigned long)pgdat->node_mem_map); 7664 #ifndef CONFIG_NUMA 7665 /* 7666 * With no DISCONTIG, the global mem_map is just set as node 0's 7667 */ 7668 if (pgdat == NODE_DATA(0)) { 7669 mem_map = NODE_DATA(0)->node_mem_map; 7670 if (page_to_pfn(mem_map) != pgdat->node_start_pfn) 7671 mem_map -= offset; 7672 } 7673 #endif 7674 } 7675 #else 7676 static inline void alloc_node_mem_map(struct pglist_data *pgdat) { } 7677 #endif /* CONFIG_FLATMEM */ 7678 7679 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 7680 static inline void pgdat_set_deferred_range(pg_data_t *pgdat) 7681 { 7682 pgdat->first_deferred_pfn = ULONG_MAX; 7683 } 7684 #else 7685 static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {} 7686 #endif 7687 7688 static void __init free_area_init_node(int nid) 7689 { 7690 pg_data_t *pgdat = NODE_DATA(nid); 7691 unsigned long start_pfn = 0; 7692 unsigned long end_pfn = 0; 7693 7694 /* pg_data_t should be reset to zero when it's allocated */ 7695 WARN_ON(pgdat->nr_zones || pgdat->kswapd_highest_zoneidx); 7696 7697 get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); 7698 7699 pgdat->node_id = nid; 7700 pgdat->node_start_pfn = start_pfn; 7701 pgdat->per_cpu_nodestats = NULL; 7702 7703 if (start_pfn != end_pfn) { 7704 pr_info("Initmem setup node %d [mem %#018Lx-%#018Lx]\n", nid, 7705 (u64)start_pfn << PAGE_SHIFT, 7706 end_pfn ? ((u64)end_pfn << PAGE_SHIFT) - 1 : 0); 7707 } else { 7708 pr_info("Initmem setup node %d as memoryless\n", nid); 7709 } 7710 7711 calculate_node_totalpages(pgdat, start_pfn, end_pfn); 7712 7713 alloc_node_mem_map(pgdat); 7714 pgdat_set_deferred_range(pgdat); 7715 7716 free_area_init_core(pgdat); 7717 } 7718 7719 static void __init free_area_init_memoryless_node(int nid) 7720 { 7721 free_area_init_node(nid); 7722 } 7723 7724 #if MAX_NUMNODES > 1 7725 /* 7726 * Figure out the number of possible node ids. 7727 */ 7728 void __init setup_nr_node_ids(void) 7729 { 7730 unsigned int highest; 7731 7732 highest = find_last_bit(node_possible_map.bits, MAX_NUMNODES); 7733 nr_node_ids = highest + 1; 7734 } 7735 #endif 7736 7737 /** 7738 * node_map_pfn_alignment - determine the maximum internode alignment 7739 * 7740 * This function should be called after node map is populated and sorted. 7741 * It calculates the maximum power of two alignment which can distinguish 7742 * all the nodes. 7743 * 7744 * For example, if all nodes are 1GiB and aligned to 1GiB, the return value 7745 * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)). If the 7746 * nodes are shifted by 256MiB, 256MiB. Note that if only the last node is 7747 * shifted, 1GiB is enough and this function will indicate so. 7748 * 7749 * This is used to test whether pfn -> nid mapping of the chosen memory 7750 * model has fine enough granularity to avoid incorrect mapping for the 7751 * populated node map. 7752 * 7753 * Return: the determined alignment in pfn's. 0 if there is no alignment 7754 * requirement (single node). 7755 */ 7756 unsigned long __init node_map_pfn_alignment(void) 7757 { 7758 unsigned long accl_mask = 0, last_end = 0; 7759 unsigned long start, end, mask; 7760 int last_nid = NUMA_NO_NODE; 7761 int i, nid; 7762 7763 for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) { 7764 if (!start || last_nid < 0 || last_nid == nid) { 7765 last_nid = nid; 7766 last_end = end; 7767 continue; 7768 } 7769 7770 /* 7771 * Start with a mask granular enough to pin-point to the 7772 * start pfn and tick off bits one-by-one until it becomes 7773 * too coarse to separate the current node from the last. 7774 */ 7775 mask = ~((1 << __ffs(start)) - 1); 7776 while (mask && last_end <= (start & (mask << 1))) 7777 mask <<= 1; 7778 7779 /* accumulate all internode masks */ 7780 accl_mask |= mask; 7781 } 7782 7783 /* convert mask to number of pages */ 7784 return ~accl_mask + 1; 7785 } 7786 7787 /** 7788 * find_min_pfn_with_active_regions - Find the minimum PFN registered 7789 * 7790 * Return: the minimum PFN based on information provided via 7791 * memblock_set_node(). 7792 */ 7793 unsigned long __init find_min_pfn_with_active_regions(void) 7794 { 7795 return PHYS_PFN(memblock_start_of_DRAM()); 7796 } 7797 7798 /* 7799 * early_calculate_totalpages() 7800 * Sum pages in active regions for movable zone. 7801 * Populate N_MEMORY for calculating usable_nodes. 7802 */ 7803 static unsigned long __init early_calculate_totalpages(void) 7804 { 7805 unsigned long totalpages = 0; 7806 unsigned long start_pfn, end_pfn; 7807 int i, nid; 7808 7809 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 7810 unsigned long pages = end_pfn - start_pfn; 7811 7812 totalpages += pages; 7813 if (pages) 7814 node_set_state(nid, N_MEMORY); 7815 } 7816 return totalpages; 7817 } 7818 7819 /* 7820 * Find the PFN the Movable zone begins in each node. Kernel memory 7821 * is spread evenly between nodes as long as the nodes have enough 7822 * memory. When they don't, some nodes will have more kernelcore than 7823 * others 7824 */ 7825 static void __init find_zone_movable_pfns_for_nodes(void) 7826 { 7827 int i, nid; 7828 unsigned long usable_startpfn; 7829 unsigned long kernelcore_node, kernelcore_remaining; 7830 /* save the state before borrow the nodemask */ 7831 nodemask_t saved_node_state = node_states[N_MEMORY]; 7832 unsigned long totalpages = early_calculate_totalpages(); 7833 int usable_nodes = nodes_weight(node_states[N_MEMORY]); 7834 struct memblock_region *r; 7835 7836 /* Need to find movable_zone earlier when movable_node is specified. */ 7837 find_usable_zone_for_movable(); 7838 7839 /* 7840 * If movable_node is specified, ignore kernelcore and movablecore 7841 * options. 7842 */ 7843 if (movable_node_is_enabled()) { 7844 for_each_mem_region(r) { 7845 if (!memblock_is_hotpluggable(r)) 7846 continue; 7847 7848 nid = memblock_get_region_node(r); 7849 7850 usable_startpfn = PFN_DOWN(r->base); 7851 zone_movable_pfn[nid] = zone_movable_pfn[nid] ? 7852 min(usable_startpfn, zone_movable_pfn[nid]) : 7853 usable_startpfn; 7854 } 7855 7856 goto out2; 7857 } 7858 7859 /* 7860 * If kernelcore=mirror is specified, ignore movablecore option 7861 */ 7862 if (mirrored_kernelcore) { 7863 bool mem_below_4gb_not_mirrored = false; 7864 7865 for_each_mem_region(r) { 7866 if (memblock_is_mirror(r)) 7867 continue; 7868 7869 nid = memblock_get_region_node(r); 7870 7871 usable_startpfn = memblock_region_memory_base_pfn(r); 7872 7873 if (usable_startpfn < 0x100000) { 7874 mem_below_4gb_not_mirrored = true; 7875 continue; 7876 } 7877 7878 zone_movable_pfn[nid] = zone_movable_pfn[nid] ? 7879 min(usable_startpfn, zone_movable_pfn[nid]) : 7880 usable_startpfn; 7881 } 7882 7883 if (mem_below_4gb_not_mirrored) 7884 pr_warn("This configuration results in unmirrored kernel memory.\n"); 7885 7886 goto out2; 7887 } 7888 7889 /* 7890 * If kernelcore=nn% or movablecore=nn% was specified, calculate the 7891 * amount of necessary memory. 7892 */ 7893 if (required_kernelcore_percent) 7894 required_kernelcore = (totalpages * 100 * required_kernelcore_percent) / 7895 10000UL; 7896 if (required_movablecore_percent) 7897 required_movablecore = (totalpages * 100 * required_movablecore_percent) / 7898 10000UL; 7899 7900 /* 7901 * If movablecore= was specified, calculate what size of 7902 * kernelcore that corresponds so that memory usable for 7903 * any allocation type is evenly spread. If both kernelcore 7904 * and movablecore are specified, then the value of kernelcore 7905 * will be used for required_kernelcore if it's greater than 7906 * what movablecore would have allowed. 7907 */ 7908 if (required_movablecore) { 7909 unsigned long corepages; 7910 7911 /* 7912 * Round-up so that ZONE_MOVABLE is at least as large as what 7913 * was requested by the user 7914 */ 7915 required_movablecore = 7916 roundup(required_movablecore, MAX_ORDER_NR_PAGES); 7917 required_movablecore = min(totalpages, required_movablecore); 7918 corepages = totalpages - required_movablecore; 7919 7920 required_kernelcore = max(required_kernelcore, corepages); 7921 } 7922 7923 /* 7924 * If kernelcore was not specified or kernelcore size is larger 7925 * than totalpages, there is no ZONE_MOVABLE. 7926 */ 7927 if (!required_kernelcore || required_kernelcore >= totalpages) 7928 goto out; 7929 7930 /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */ 7931 usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone]; 7932 7933 restart: 7934 /* Spread kernelcore memory as evenly as possible throughout nodes */ 7935 kernelcore_node = required_kernelcore / usable_nodes; 7936 for_each_node_state(nid, N_MEMORY) { 7937 unsigned long start_pfn, end_pfn; 7938 7939 /* 7940 * Recalculate kernelcore_node if the division per node 7941 * now exceeds what is necessary to satisfy the requested 7942 * amount of memory for the kernel 7943 */ 7944 if (required_kernelcore < kernelcore_node) 7945 kernelcore_node = required_kernelcore / usable_nodes; 7946 7947 /* 7948 * As the map is walked, we track how much memory is usable 7949 * by the kernel using kernelcore_remaining. When it is 7950 * 0, the rest of the node is usable by ZONE_MOVABLE 7951 */ 7952 kernelcore_remaining = kernelcore_node; 7953 7954 /* Go through each range of PFNs within this node */ 7955 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { 7956 unsigned long size_pages; 7957 7958 start_pfn = max(start_pfn, zone_movable_pfn[nid]); 7959 if (start_pfn >= end_pfn) 7960 continue; 7961 7962 /* Account for what is only usable for kernelcore */ 7963 if (start_pfn < usable_startpfn) { 7964 unsigned long kernel_pages; 7965 kernel_pages = min(end_pfn, usable_startpfn) 7966 - start_pfn; 7967 7968 kernelcore_remaining -= min(kernel_pages, 7969 kernelcore_remaining); 7970 required_kernelcore -= min(kernel_pages, 7971 required_kernelcore); 7972 7973 /* Continue if range is now fully accounted */ 7974 if (end_pfn <= usable_startpfn) { 7975 7976 /* 7977 * Push zone_movable_pfn to the end so 7978 * that if we have to rebalance 7979 * kernelcore across nodes, we will 7980 * not double account here 7981 */ 7982 zone_movable_pfn[nid] = end_pfn; 7983 continue; 7984 } 7985 start_pfn = usable_startpfn; 7986 } 7987 7988 /* 7989 * The usable PFN range for ZONE_MOVABLE is from 7990 * start_pfn->end_pfn. Calculate size_pages as the 7991 * number of pages used as kernelcore 7992 */ 7993 size_pages = end_pfn - start_pfn; 7994 if (size_pages > kernelcore_remaining) 7995 size_pages = kernelcore_remaining; 7996 zone_movable_pfn[nid] = start_pfn + size_pages; 7997 7998 /* 7999 * Some kernelcore has been met, update counts and 8000 * break if the kernelcore for this node has been 8001 * satisfied 8002 */ 8003 required_kernelcore -= min(required_kernelcore, 8004 size_pages); 8005 kernelcore_remaining -= size_pages; 8006 if (!kernelcore_remaining) 8007 break; 8008 } 8009 } 8010 8011 /* 8012 * If there is still required_kernelcore, we do another pass with one 8013 * less node in the count. This will push zone_movable_pfn[nid] further 8014 * along on the nodes that still have memory until kernelcore is 8015 * satisfied 8016 */ 8017 usable_nodes--; 8018 if (usable_nodes && required_kernelcore > usable_nodes) 8019 goto restart; 8020 8021 out2: 8022 /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */ 8023 for (nid = 0; nid < MAX_NUMNODES; nid++) { 8024 unsigned long start_pfn, end_pfn; 8025 8026 zone_movable_pfn[nid] = 8027 roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES); 8028 8029 get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); 8030 if (zone_movable_pfn[nid] >= end_pfn) 8031 zone_movable_pfn[nid] = 0; 8032 } 8033 8034 out: 8035 /* restore the node_state */ 8036 node_states[N_MEMORY] = saved_node_state; 8037 } 8038 8039 /* Any regular or high memory on that node ? */ 8040 static void check_for_memory(pg_data_t *pgdat, int nid) 8041 { 8042 enum zone_type zone_type; 8043 8044 for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) { 8045 struct zone *zone = &pgdat->node_zones[zone_type]; 8046 if (populated_zone(zone)) { 8047 if (IS_ENABLED(CONFIG_HIGHMEM)) 8048 node_set_state(nid, N_HIGH_MEMORY); 8049 if (zone_type <= ZONE_NORMAL) 8050 node_set_state(nid, N_NORMAL_MEMORY); 8051 break; 8052 } 8053 } 8054 } 8055 8056 /* 8057 * Some architectures, e.g. ARC may have ZONE_HIGHMEM below ZONE_NORMAL. For 8058 * such cases we allow max_zone_pfn sorted in the descending order 8059 */ 8060 bool __weak arch_has_descending_max_zone_pfns(void) 8061 { 8062 return false; 8063 } 8064 8065 /** 8066 * free_area_init - Initialise all pg_data_t and zone data 8067 * @max_zone_pfn: an array of max PFNs for each zone 8068 * 8069 * This will call free_area_init_node() for each active node in the system. 8070 * Using the page ranges provided by memblock_set_node(), the size of each 8071 * zone in each node and their holes is calculated. If the maximum PFN 8072 * between two adjacent zones match, it is assumed that the zone is empty. 8073 * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed 8074 * that arch_max_dma32_pfn has no pages. It is also assumed that a zone 8075 * starts where the previous one ended. For example, ZONE_DMA32 starts 8076 * at arch_max_dma_pfn. 8077 */ 8078 void __init free_area_init(unsigned long *max_zone_pfn) 8079 { 8080 unsigned long start_pfn, end_pfn; 8081 int i, nid, zone; 8082 bool descending; 8083 8084 /* Record where the zone boundaries are */ 8085 memset(arch_zone_lowest_possible_pfn, 0, 8086 sizeof(arch_zone_lowest_possible_pfn)); 8087 memset(arch_zone_highest_possible_pfn, 0, 8088 sizeof(arch_zone_highest_possible_pfn)); 8089 8090 start_pfn = find_min_pfn_with_active_regions(); 8091 descending = arch_has_descending_max_zone_pfns(); 8092 8093 for (i = 0; i < MAX_NR_ZONES; i++) { 8094 if (descending) 8095 zone = MAX_NR_ZONES - i - 1; 8096 else 8097 zone = i; 8098 8099 if (zone == ZONE_MOVABLE) 8100 continue; 8101 8102 end_pfn = max(max_zone_pfn[zone], start_pfn); 8103 arch_zone_lowest_possible_pfn[zone] = start_pfn; 8104 arch_zone_highest_possible_pfn[zone] = end_pfn; 8105 8106 start_pfn = end_pfn; 8107 } 8108 8109 /* Find the PFNs that ZONE_MOVABLE begins at in each node */ 8110 memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn)); 8111 find_zone_movable_pfns_for_nodes(); 8112 8113 /* Print out the zone ranges */ 8114 pr_info("Zone ranges:\n"); 8115 for (i = 0; i < MAX_NR_ZONES; i++) { 8116 if (i == ZONE_MOVABLE) 8117 continue; 8118 pr_info(" %-8s ", zone_names[i]); 8119 if (arch_zone_lowest_possible_pfn[i] == 8120 arch_zone_highest_possible_pfn[i]) 8121 pr_cont("empty\n"); 8122 else 8123 pr_cont("[mem %#018Lx-%#018Lx]\n", 8124 (u64)arch_zone_lowest_possible_pfn[i] 8125 << PAGE_SHIFT, 8126 ((u64)arch_zone_highest_possible_pfn[i] 8127 << PAGE_SHIFT) - 1); 8128 } 8129 8130 /* Print out the PFNs ZONE_MOVABLE begins at in each node */ 8131 pr_info("Movable zone start for each node\n"); 8132 for (i = 0; i < MAX_NUMNODES; i++) { 8133 if (zone_movable_pfn[i]) 8134 pr_info(" Node %d: %#018Lx\n", i, 8135 (u64)zone_movable_pfn[i] << PAGE_SHIFT); 8136 } 8137 8138 /* 8139 * Print out the early node map, and initialize the 8140 * subsection-map relative to active online memory ranges to 8141 * enable future "sub-section" extensions of the memory map. 8142 */ 8143 pr_info("Early memory node ranges\n"); 8144 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 8145 pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid, 8146 (u64)start_pfn << PAGE_SHIFT, 8147 ((u64)end_pfn << PAGE_SHIFT) - 1); 8148 subsection_map_init(start_pfn, end_pfn - start_pfn); 8149 } 8150 8151 /* Initialise every node */ 8152 mminit_verify_pageflags_layout(); 8153 setup_nr_node_ids(); 8154 for_each_node(nid) { 8155 pg_data_t *pgdat; 8156 8157 if (!node_online(nid)) { 8158 pr_info("Initializing node %d as memoryless\n", nid); 8159 8160 /* Allocator not initialized yet */ 8161 pgdat = arch_alloc_nodedata(nid); 8162 if (!pgdat) { 8163 pr_err("Cannot allocate %zuB for node %d.\n", 8164 sizeof(*pgdat), nid); 8165 continue; 8166 } 8167 arch_refresh_nodedata(nid, pgdat); 8168 free_area_init_memoryless_node(nid); 8169 8170 /* 8171 * We do not want to confuse userspace by sysfs 8172 * files/directories for node without any memory 8173 * attached to it, so this node is not marked as 8174 * N_MEMORY and not marked online so that no sysfs 8175 * hierarchy will be created via register_one_node for 8176 * it. The pgdat will get fully initialized by 8177 * hotadd_init_pgdat() when memory is hotplugged into 8178 * this node. 8179 */ 8180 continue; 8181 } 8182 8183 pgdat = NODE_DATA(nid); 8184 free_area_init_node(nid); 8185 8186 /* Any memory on that node */ 8187 if (pgdat->node_present_pages) 8188 node_set_state(nid, N_MEMORY); 8189 check_for_memory(pgdat, nid); 8190 } 8191 8192 memmap_init(); 8193 } 8194 8195 static int __init cmdline_parse_core(char *p, unsigned long *core, 8196 unsigned long *percent) 8197 { 8198 unsigned long long coremem; 8199 char *endptr; 8200 8201 if (!p) 8202 return -EINVAL; 8203 8204 /* Value may be a percentage of total memory, otherwise bytes */ 8205 coremem = simple_strtoull(p, &endptr, 0); 8206 if (*endptr == '%') { 8207 /* Paranoid check for percent values greater than 100 */ 8208 WARN_ON(coremem > 100); 8209 8210 *percent = coremem; 8211 } else { 8212 coremem = memparse(p, &p); 8213 /* Paranoid check that UL is enough for the coremem value */ 8214 WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX); 8215 8216 *core = coremem >> PAGE_SHIFT; 8217 *percent = 0UL; 8218 } 8219 return 0; 8220 } 8221 8222 /* 8223 * kernelcore=size sets the amount of memory for use for allocations that 8224 * cannot be reclaimed or migrated. 8225 */ 8226 static int __init cmdline_parse_kernelcore(char *p) 8227 { 8228 /* parse kernelcore=mirror */ 8229 if (parse_option_str(p, "mirror")) { 8230 mirrored_kernelcore = true; 8231 return 0; 8232 } 8233 8234 return cmdline_parse_core(p, &required_kernelcore, 8235 &required_kernelcore_percent); 8236 } 8237 8238 /* 8239 * movablecore=size sets the amount of memory for use for allocations that 8240 * can be reclaimed or migrated. 8241 */ 8242 static int __init cmdline_parse_movablecore(char *p) 8243 { 8244 return cmdline_parse_core(p, &required_movablecore, 8245 &required_movablecore_percent); 8246 } 8247 8248 early_param("kernelcore", cmdline_parse_kernelcore); 8249 early_param("movablecore", cmdline_parse_movablecore); 8250 8251 void adjust_managed_page_count(struct page *page, long count) 8252 { 8253 atomic_long_add(count, &page_zone(page)->managed_pages); 8254 totalram_pages_add(count); 8255 #ifdef CONFIG_HIGHMEM 8256 if (PageHighMem(page)) 8257 totalhigh_pages_add(count); 8258 #endif 8259 } 8260 EXPORT_SYMBOL(adjust_managed_page_count); 8261 8262 unsigned long free_reserved_area(void *start, void *end, int poison, const char *s) 8263 { 8264 void *pos; 8265 unsigned long pages = 0; 8266 8267 start = (void *)PAGE_ALIGN((unsigned long)start); 8268 end = (void *)((unsigned long)end & PAGE_MASK); 8269 for (pos = start; pos < end; pos += PAGE_SIZE, pages++) { 8270 struct page *page = virt_to_page(pos); 8271 void *direct_map_addr; 8272 8273 /* 8274 * 'direct_map_addr' might be different from 'pos' 8275 * because some architectures' virt_to_page() 8276 * work with aliases. Getting the direct map 8277 * address ensures that we get a _writeable_ 8278 * alias for the memset(). 8279 */ 8280 direct_map_addr = page_address(page); 8281 /* 8282 * Perform a kasan-unchecked memset() since this memory 8283 * has not been initialized. 8284 */ 8285 direct_map_addr = kasan_reset_tag(direct_map_addr); 8286 if ((unsigned int)poison <= 0xFF) 8287 memset(direct_map_addr, poison, PAGE_SIZE); 8288 8289 free_reserved_page(page); 8290 } 8291 8292 if (pages && s) 8293 pr_info("Freeing %s memory: %ldK\n", s, K(pages)); 8294 8295 return pages; 8296 } 8297 8298 void __init mem_init_print_info(void) 8299 { 8300 unsigned long physpages, codesize, datasize, rosize, bss_size; 8301 unsigned long init_code_size, init_data_size; 8302 8303 physpages = get_num_physpages(); 8304 codesize = _etext - _stext; 8305 datasize = _edata - _sdata; 8306 rosize = __end_rodata - __start_rodata; 8307 bss_size = __bss_stop - __bss_start; 8308 init_data_size = __init_end - __init_begin; 8309 init_code_size = _einittext - _sinittext; 8310 8311 /* 8312 * Detect special cases and adjust section sizes accordingly: 8313 * 1) .init.* may be embedded into .data sections 8314 * 2) .init.text.* may be out of [__init_begin, __init_end], 8315 * please refer to arch/tile/kernel/vmlinux.lds.S. 8316 * 3) .rodata.* may be embedded into .text or .data sections. 8317 */ 8318 #define adj_init_size(start, end, size, pos, adj) \ 8319 do { \ 8320 if (&start[0] <= &pos[0] && &pos[0] < &end[0] && size > adj) \ 8321 size -= adj; \ 8322 } while (0) 8323 8324 adj_init_size(__init_begin, __init_end, init_data_size, 8325 _sinittext, init_code_size); 8326 adj_init_size(_stext, _etext, codesize, _sinittext, init_code_size); 8327 adj_init_size(_sdata, _edata, datasize, __init_begin, init_data_size); 8328 adj_init_size(_stext, _etext, codesize, __start_rodata, rosize); 8329 adj_init_size(_sdata, _edata, datasize, __start_rodata, rosize); 8330 8331 #undef adj_init_size 8332 8333 pr_info("Memory: %luK/%luK available (%luK kernel code, %luK rwdata, %luK rodata, %luK init, %luK bss, %luK reserved, %luK cma-reserved" 8334 #ifdef CONFIG_HIGHMEM 8335 ", %luK highmem" 8336 #endif 8337 ")\n", 8338 K(nr_free_pages()), K(physpages), 8339 codesize >> 10, datasize >> 10, rosize >> 10, 8340 (init_data_size + init_code_size) >> 10, bss_size >> 10, 8341 K(physpages - totalram_pages() - totalcma_pages), 8342 K(totalcma_pages) 8343 #ifdef CONFIG_HIGHMEM 8344 , K(totalhigh_pages()) 8345 #endif 8346 ); 8347 } 8348 8349 /** 8350 * set_dma_reserve - set the specified number of pages reserved in the first zone 8351 * @new_dma_reserve: The number of pages to mark reserved 8352 * 8353 * The per-cpu batchsize and zone watermarks are determined by managed_pages. 8354 * In the DMA zone, a significant percentage may be consumed by kernel image 8355 * and other unfreeable allocations which can skew the watermarks badly. This 8356 * function may optionally be used to account for unfreeable pages in the 8357 * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and 8358 * smaller per-cpu batchsize. 8359 */ 8360 void __init set_dma_reserve(unsigned long new_dma_reserve) 8361 { 8362 dma_reserve = new_dma_reserve; 8363 } 8364 8365 static int page_alloc_cpu_dead(unsigned int cpu) 8366 { 8367 struct zone *zone; 8368 8369 lru_add_drain_cpu(cpu); 8370 mlock_page_drain_remote(cpu); 8371 drain_pages(cpu); 8372 8373 /* 8374 * Spill the event counters of the dead processor 8375 * into the current processors event counters. 8376 * This artificially elevates the count of the current 8377 * processor. 8378 */ 8379 vm_events_fold_cpu(cpu); 8380 8381 /* 8382 * Zero the differential counters of the dead processor 8383 * so that the vm statistics are consistent. 8384 * 8385 * This is only okay since the processor is dead and cannot 8386 * race with what we are doing. 8387 */ 8388 cpu_vm_stats_fold(cpu); 8389 8390 for_each_populated_zone(zone) 8391 zone_pcp_update(zone, 0); 8392 8393 return 0; 8394 } 8395 8396 static int page_alloc_cpu_online(unsigned int cpu) 8397 { 8398 struct zone *zone; 8399 8400 for_each_populated_zone(zone) 8401 zone_pcp_update(zone, 1); 8402 return 0; 8403 } 8404 8405 #ifdef CONFIG_NUMA 8406 int hashdist = HASHDIST_DEFAULT; 8407 8408 static int __init set_hashdist(char *str) 8409 { 8410 if (!str) 8411 return 0; 8412 hashdist = simple_strtoul(str, &str, 0); 8413 return 1; 8414 } 8415 __setup("hashdist=", set_hashdist); 8416 #endif 8417 8418 void __init page_alloc_init(void) 8419 { 8420 int ret; 8421 8422 #ifdef CONFIG_NUMA 8423 if (num_node_state(N_MEMORY) == 1) 8424 hashdist = 0; 8425 #endif 8426 8427 ret = cpuhp_setup_state_nocalls(CPUHP_PAGE_ALLOC, 8428 "mm/page_alloc:pcp", 8429 page_alloc_cpu_online, 8430 page_alloc_cpu_dead); 8431 WARN_ON(ret < 0); 8432 } 8433 8434 /* 8435 * calculate_totalreserve_pages - called when sysctl_lowmem_reserve_ratio 8436 * or min_free_kbytes changes. 8437 */ 8438 static void calculate_totalreserve_pages(void) 8439 { 8440 struct pglist_data *pgdat; 8441 unsigned long reserve_pages = 0; 8442 enum zone_type i, j; 8443 8444 for_each_online_pgdat(pgdat) { 8445 8446 pgdat->totalreserve_pages = 0; 8447 8448 for (i = 0; i < MAX_NR_ZONES; i++) { 8449 struct zone *zone = pgdat->node_zones + i; 8450 long max = 0; 8451 unsigned long managed_pages = zone_managed_pages(zone); 8452 8453 /* Find valid and maximum lowmem_reserve in the zone */ 8454 for (j = i; j < MAX_NR_ZONES; j++) { 8455 if (zone->lowmem_reserve[j] > max) 8456 max = zone->lowmem_reserve[j]; 8457 } 8458 8459 /* we treat the high watermark as reserved pages. */ 8460 max += high_wmark_pages(zone); 8461 8462 if (max > managed_pages) 8463 max = managed_pages; 8464 8465 pgdat->totalreserve_pages += max; 8466 8467 reserve_pages += max; 8468 } 8469 } 8470 totalreserve_pages = reserve_pages; 8471 } 8472 8473 /* 8474 * setup_per_zone_lowmem_reserve - called whenever 8475 * sysctl_lowmem_reserve_ratio changes. Ensures that each zone 8476 * has a correct pages reserved value, so an adequate number of 8477 * pages are left in the zone after a successful __alloc_pages(). 8478 */ 8479 static void setup_per_zone_lowmem_reserve(void) 8480 { 8481 struct pglist_data *pgdat; 8482 enum zone_type i, j; 8483 8484 for_each_online_pgdat(pgdat) { 8485 for (i = 0; i < MAX_NR_ZONES - 1; i++) { 8486 struct zone *zone = &pgdat->node_zones[i]; 8487 int ratio = sysctl_lowmem_reserve_ratio[i]; 8488 bool clear = !ratio || !zone_managed_pages(zone); 8489 unsigned long managed_pages = 0; 8490 8491 for (j = i + 1; j < MAX_NR_ZONES; j++) { 8492 struct zone *upper_zone = &pgdat->node_zones[j]; 8493 8494 managed_pages += zone_managed_pages(upper_zone); 8495 8496 if (clear) 8497 zone->lowmem_reserve[j] = 0; 8498 else 8499 zone->lowmem_reserve[j] = managed_pages / ratio; 8500 } 8501 } 8502 } 8503 8504 /* update totalreserve_pages */ 8505 calculate_totalreserve_pages(); 8506 } 8507 8508 static void __setup_per_zone_wmarks(void) 8509 { 8510 unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10); 8511 unsigned long lowmem_pages = 0; 8512 struct zone *zone; 8513 unsigned long flags; 8514 8515 /* Calculate total number of !ZONE_HIGHMEM pages */ 8516 for_each_zone(zone) { 8517 if (!is_highmem(zone)) 8518 lowmem_pages += zone_managed_pages(zone); 8519 } 8520 8521 for_each_zone(zone) { 8522 u64 tmp; 8523 8524 spin_lock_irqsave(&zone->lock, flags); 8525 tmp = (u64)pages_min * zone_managed_pages(zone); 8526 do_div(tmp, lowmem_pages); 8527 if (is_highmem(zone)) { 8528 /* 8529 * __GFP_HIGH and PF_MEMALLOC allocations usually don't 8530 * need highmem pages, so cap pages_min to a small 8531 * value here. 8532 * 8533 * The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN) 8534 * deltas control async page reclaim, and so should 8535 * not be capped for highmem. 8536 */ 8537 unsigned long min_pages; 8538 8539 min_pages = zone_managed_pages(zone) / 1024; 8540 min_pages = clamp(min_pages, SWAP_CLUSTER_MAX, 128UL); 8541 zone->_watermark[WMARK_MIN] = min_pages; 8542 } else { 8543 /* 8544 * If it's a lowmem zone, reserve a number of pages 8545 * proportionate to the zone's size. 8546 */ 8547 zone->_watermark[WMARK_MIN] = tmp; 8548 } 8549 8550 /* 8551 * Set the kswapd watermarks distance according to the 8552 * scale factor in proportion to available memory, but 8553 * ensure a minimum size on small systems. 8554 */ 8555 tmp = max_t(u64, tmp >> 2, 8556 mult_frac(zone_managed_pages(zone), 8557 watermark_scale_factor, 10000)); 8558 8559 zone->watermark_boost = 0; 8560 zone->_watermark[WMARK_LOW] = min_wmark_pages(zone) + tmp; 8561 zone->_watermark[WMARK_HIGH] = low_wmark_pages(zone) + tmp; 8562 zone->_watermark[WMARK_PROMO] = high_wmark_pages(zone) + tmp; 8563 8564 spin_unlock_irqrestore(&zone->lock, flags); 8565 } 8566 8567 /* update totalreserve_pages */ 8568 calculate_totalreserve_pages(); 8569 } 8570 8571 /** 8572 * setup_per_zone_wmarks - called when min_free_kbytes changes 8573 * or when memory is hot-{added|removed} 8574 * 8575 * Ensures that the watermark[min,low,high] values for each zone are set 8576 * correctly with respect to min_free_kbytes. 8577 */ 8578 void setup_per_zone_wmarks(void) 8579 { 8580 struct zone *zone; 8581 static DEFINE_SPINLOCK(lock); 8582 8583 spin_lock(&lock); 8584 __setup_per_zone_wmarks(); 8585 spin_unlock(&lock); 8586 8587 /* 8588 * The watermark size have changed so update the pcpu batch 8589 * and high limits or the limits may be inappropriate. 8590 */ 8591 for_each_zone(zone) 8592 zone_pcp_update(zone, 0); 8593 } 8594 8595 /* 8596 * Initialise min_free_kbytes. 8597 * 8598 * For small machines we want it small (128k min). For large machines 8599 * we want it large (256MB max). But it is not linear, because network 8600 * bandwidth does not increase linearly with machine size. We use 8601 * 8602 * min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy: 8603 * min_free_kbytes = sqrt(lowmem_kbytes * 16) 8604 * 8605 * which yields 8606 * 8607 * 16MB: 512k 8608 * 32MB: 724k 8609 * 64MB: 1024k 8610 * 128MB: 1448k 8611 * 256MB: 2048k 8612 * 512MB: 2896k 8613 * 1024MB: 4096k 8614 * 2048MB: 5792k 8615 * 4096MB: 8192k 8616 * 8192MB: 11584k 8617 * 16384MB: 16384k 8618 */ 8619 void calculate_min_free_kbytes(void) 8620 { 8621 unsigned long lowmem_kbytes; 8622 int new_min_free_kbytes; 8623 8624 lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10); 8625 new_min_free_kbytes = int_sqrt(lowmem_kbytes * 16); 8626 8627 if (new_min_free_kbytes > user_min_free_kbytes) 8628 min_free_kbytes = clamp(new_min_free_kbytes, 128, 262144); 8629 else 8630 pr_warn("min_free_kbytes is not updated to %d because user defined value %d is preferred\n", 8631 new_min_free_kbytes, user_min_free_kbytes); 8632 8633 } 8634 8635 int __meminit init_per_zone_wmark_min(void) 8636 { 8637 calculate_min_free_kbytes(); 8638 setup_per_zone_wmarks(); 8639 refresh_zone_stat_thresholds(); 8640 setup_per_zone_lowmem_reserve(); 8641 8642 #ifdef CONFIG_NUMA 8643 setup_min_unmapped_ratio(); 8644 setup_min_slab_ratio(); 8645 #endif 8646 8647 khugepaged_min_free_kbytes_update(); 8648 8649 return 0; 8650 } 8651 postcore_initcall(init_per_zone_wmark_min) 8652 8653 /* 8654 * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so 8655 * that we can call two helper functions whenever min_free_kbytes 8656 * changes. 8657 */ 8658 int min_free_kbytes_sysctl_handler(struct ctl_table *table, int write, 8659 void *buffer, size_t *length, loff_t *ppos) 8660 { 8661 int rc; 8662 8663 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 8664 if (rc) 8665 return rc; 8666 8667 if (write) { 8668 user_min_free_kbytes = min_free_kbytes; 8669 setup_per_zone_wmarks(); 8670 } 8671 return 0; 8672 } 8673 8674 int watermark_scale_factor_sysctl_handler(struct ctl_table *table, int write, 8675 void *buffer, size_t *length, loff_t *ppos) 8676 { 8677 int rc; 8678 8679 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 8680 if (rc) 8681 return rc; 8682 8683 if (write) 8684 setup_per_zone_wmarks(); 8685 8686 return 0; 8687 } 8688 8689 #ifdef CONFIG_NUMA 8690 static void setup_min_unmapped_ratio(void) 8691 { 8692 pg_data_t *pgdat; 8693 struct zone *zone; 8694 8695 for_each_online_pgdat(pgdat) 8696 pgdat->min_unmapped_pages = 0; 8697 8698 for_each_zone(zone) 8699 zone->zone_pgdat->min_unmapped_pages += (zone_managed_pages(zone) * 8700 sysctl_min_unmapped_ratio) / 100; 8701 } 8702 8703 8704 int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *table, int write, 8705 void *buffer, size_t *length, loff_t *ppos) 8706 { 8707 int rc; 8708 8709 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 8710 if (rc) 8711 return rc; 8712 8713 setup_min_unmapped_ratio(); 8714 8715 return 0; 8716 } 8717 8718 static void setup_min_slab_ratio(void) 8719 { 8720 pg_data_t *pgdat; 8721 struct zone *zone; 8722 8723 for_each_online_pgdat(pgdat) 8724 pgdat->min_slab_pages = 0; 8725 8726 for_each_zone(zone) 8727 zone->zone_pgdat->min_slab_pages += (zone_managed_pages(zone) * 8728 sysctl_min_slab_ratio) / 100; 8729 } 8730 8731 int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *table, int write, 8732 void *buffer, size_t *length, loff_t *ppos) 8733 { 8734 int rc; 8735 8736 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 8737 if (rc) 8738 return rc; 8739 8740 setup_min_slab_ratio(); 8741 8742 return 0; 8743 } 8744 #endif 8745 8746 /* 8747 * lowmem_reserve_ratio_sysctl_handler - just a wrapper around 8748 * proc_dointvec() so that we can call setup_per_zone_lowmem_reserve() 8749 * whenever sysctl_lowmem_reserve_ratio changes. 8750 * 8751 * The reserve ratio obviously has absolutely no relation with the 8752 * minimum watermarks. The lowmem reserve ratio can only make sense 8753 * if in function of the boot time zone sizes. 8754 */ 8755 int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *table, int write, 8756 void *buffer, size_t *length, loff_t *ppos) 8757 { 8758 int i; 8759 8760 proc_dointvec_minmax(table, write, buffer, length, ppos); 8761 8762 for (i = 0; i < MAX_NR_ZONES; i++) { 8763 if (sysctl_lowmem_reserve_ratio[i] < 1) 8764 sysctl_lowmem_reserve_ratio[i] = 0; 8765 } 8766 8767 setup_per_zone_lowmem_reserve(); 8768 return 0; 8769 } 8770 8771 /* 8772 * percpu_pagelist_high_fraction - changes the pcp->high for each zone on each 8773 * cpu. It is the fraction of total pages in each zone that a hot per cpu 8774 * pagelist can have before it gets flushed back to buddy allocator. 8775 */ 8776 int percpu_pagelist_high_fraction_sysctl_handler(struct ctl_table *table, 8777 int write, void *buffer, size_t *length, loff_t *ppos) 8778 { 8779 struct zone *zone; 8780 int old_percpu_pagelist_high_fraction; 8781 int ret; 8782 8783 mutex_lock(&pcp_batch_high_lock); 8784 old_percpu_pagelist_high_fraction = percpu_pagelist_high_fraction; 8785 8786 ret = proc_dointvec_minmax(table, write, buffer, length, ppos); 8787 if (!write || ret < 0) 8788 goto out; 8789 8790 /* Sanity checking to avoid pcp imbalance */ 8791 if (percpu_pagelist_high_fraction && 8792 percpu_pagelist_high_fraction < MIN_PERCPU_PAGELIST_HIGH_FRACTION) { 8793 percpu_pagelist_high_fraction = old_percpu_pagelist_high_fraction; 8794 ret = -EINVAL; 8795 goto out; 8796 } 8797 8798 /* No change? */ 8799 if (percpu_pagelist_high_fraction == old_percpu_pagelist_high_fraction) 8800 goto out; 8801 8802 for_each_populated_zone(zone) 8803 zone_set_pageset_high_and_batch(zone, 0); 8804 out: 8805 mutex_unlock(&pcp_batch_high_lock); 8806 return ret; 8807 } 8808 8809 #ifndef __HAVE_ARCH_RESERVED_KERNEL_PAGES 8810 /* 8811 * Returns the number of pages that arch has reserved but 8812 * is not known to alloc_large_system_hash(). 8813 */ 8814 static unsigned long __init arch_reserved_kernel_pages(void) 8815 { 8816 return 0; 8817 } 8818 #endif 8819 8820 /* 8821 * Adaptive scale is meant to reduce sizes of hash tables on large memory 8822 * machines. As memory size is increased the scale is also increased but at 8823 * slower pace. Starting from ADAPT_SCALE_BASE (64G), every time memory 8824 * quadruples the scale is increased by one, which means the size of hash table 8825 * only doubles, instead of quadrupling as well. 8826 * Because 32-bit systems cannot have large physical memory, where this scaling 8827 * makes sense, it is disabled on such platforms. 8828 */ 8829 #if __BITS_PER_LONG > 32 8830 #define ADAPT_SCALE_BASE (64ul << 30) 8831 #define ADAPT_SCALE_SHIFT 2 8832 #define ADAPT_SCALE_NPAGES (ADAPT_SCALE_BASE >> PAGE_SHIFT) 8833 #endif 8834 8835 /* 8836 * allocate a large system hash table from bootmem 8837 * - it is assumed that the hash table must contain an exact power-of-2 8838 * quantity of entries 8839 * - limit is the number of hash buckets, not the total allocation size 8840 */ 8841 void *__init alloc_large_system_hash(const char *tablename, 8842 unsigned long bucketsize, 8843 unsigned long numentries, 8844 int scale, 8845 int flags, 8846 unsigned int *_hash_shift, 8847 unsigned int *_hash_mask, 8848 unsigned long low_limit, 8849 unsigned long high_limit) 8850 { 8851 unsigned long long max = high_limit; 8852 unsigned long log2qty, size; 8853 void *table = NULL; 8854 gfp_t gfp_flags; 8855 bool virt; 8856 bool huge; 8857 8858 /* allow the kernel cmdline to have a say */ 8859 if (!numentries) { 8860 /* round applicable memory size up to nearest megabyte */ 8861 numentries = nr_kernel_pages; 8862 numentries -= arch_reserved_kernel_pages(); 8863 8864 /* It isn't necessary when PAGE_SIZE >= 1MB */ 8865 if (PAGE_SHIFT < 20) 8866 numentries = round_up(numentries, (1<<20)/PAGE_SIZE); 8867 8868 #if __BITS_PER_LONG > 32 8869 if (!high_limit) { 8870 unsigned long adapt; 8871 8872 for (adapt = ADAPT_SCALE_NPAGES; adapt < numentries; 8873 adapt <<= ADAPT_SCALE_SHIFT) 8874 scale++; 8875 } 8876 #endif 8877 8878 /* limit to 1 bucket per 2^scale bytes of low memory */ 8879 if (scale > PAGE_SHIFT) 8880 numentries >>= (scale - PAGE_SHIFT); 8881 else 8882 numentries <<= (PAGE_SHIFT - scale); 8883 8884 /* Make sure we've got at least a 0-order allocation.. */ 8885 if (unlikely(flags & HASH_SMALL)) { 8886 /* Makes no sense without HASH_EARLY */ 8887 WARN_ON(!(flags & HASH_EARLY)); 8888 if (!(numentries >> *_hash_shift)) { 8889 numentries = 1UL << *_hash_shift; 8890 BUG_ON(!numentries); 8891 } 8892 } else if (unlikely((numentries * bucketsize) < PAGE_SIZE)) 8893 numentries = PAGE_SIZE / bucketsize; 8894 } 8895 numentries = roundup_pow_of_two(numentries); 8896 8897 /* limit allocation size to 1/16 total memory by default */ 8898 if (max == 0) { 8899 max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4; 8900 do_div(max, bucketsize); 8901 } 8902 max = min(max, 0x80000000ULL); 8903 8904 if (numentries < low_limit) 8905 numentries = low_limit; 8906 if (numentries > max) 8907 numentries = max; 8908 8909 log2qty = ilog2(numentries); 8910 8911 gfp_flags = (flags & HASH_ZERO) ? GFP_ATOMIC | __GFP_ZERO : GFP_ATOMIC; 8912 do { 8913 virt = false; 8914 size = bucketsize << log2qty; 8915 if (flags & HASH_EARLY) { 8916 if (flags & HASH_ZERO) 8917 table = memblock_alloc(size, SMP_CACHE_BYTES); 8918 else 8919 table = memblock_alloc_raw(size, 8920 SMP_CACHE_BYTES); 8921 } else if (get_order(size) >= MAX_ORDER || hashdist) { 8922 table = vmalloc_huge(size, gfp_flags); 8923 virt = true; 8924 if (table) 8925 huge = is_vm_area_hugepages(table); 8926 } else { 8927 /* 8928 * If bucketsize is not a power-of-two, we may free 8929 * some pages at the end of hash table which 8930 * alloc_pages_exact() automatically does 8931 */ 8932 table = alloc_pages_exact(size, gfp_flags); 8933 kmemleak_alloc(table, size, 1, gfp_flags); 8934 } 8935 } while (!table && size > PAGE_SIZE && --log2qty); 8936 8937 if (!table) 8938 panic("Failed to allocate %s hash table\n", tablename); 8939 8940 pr_info("%s hash table entries: %ld (order: %d, %lu bytes, %s)\n", 8941 tablename, 1UL << log2qty, ilog2(size) - PAGE_SHIFT, size, 8942 virt ? (huge ? "vmalloc hugepage" : "vmalloc") : "linear"); 8943 8944 if (_hash_shift) 8945 *_hash_shift = log2qty; 8946 if (_hash_mask) 8947 *_hash_mask = (1 << log2qty) - 1; 8948 8949 return table; 8950 } 8951 8952 /* 8953 * This function checks whether pageblock includes unmovable pages or not. 8954 * 8955 * PageLRU check without isolation or lru_lock could race so that 8956 * MIGRATE_MOVABLE block might include unmovable pages. And __PageMovable 8957 * check without lock_page also may miss some movable non-lru pages at 8958 * race condition. So you can't expect this function should be exact. 8959 * 8960 * Returns a page without holding a reference. If the caller wants to 8961 * dereference that page (e.g., dumping), it has to make sure that it 8962 * cannot get removed (e.g., via memory unplug) concurrently. 8963 * 8964 */ 8965 struct page *has_unmovable_pages(struct zone *zone, struct page *page, 8966 int migratetype, int flags) 8967 { 8968 unsigned long iter = 0; 8969 unsigned long pfn = page_to_pfn(page); 8970 unsigned long offset = pfn % pageblock_nr_pages; 8971 8972 if (is_migrate_cma_page(page)) { 8973 /* 8974 * CMA allocations (alloc_contig_range) really need to mark 8975 * isolate CMA pageblocks even when they are not movable in fact 8976 * so consider them movable here. 8977 */ 8978 if (is_migrate_cma(migratetype)) 8979 return NULL; 8980 8981 return page; 8982 } 8983 8984 for (; iter < pageblock_nr_pages - offset; iter++) { 8985 page = pfn_to_page(pfn + iter); 8986 8987 /* 8988 * Both, bootmem allocations and memory holes are marked 8989 * PG_reserved and are unmovable. We can even have unmovable 8990 * allocations inside ZONE_MOVABLE, for example when 8991 * specifying "movablecore". 8992 */ 8993 if (PageReserved(page)) 8994 return page; 8995 8996 /* 8997 * If the zone is movable and we have ruled out all reserved 8998 * pages then it should be reasonably safe to assume the rest 8999 * is movable. 9000 */ 9001 if (zone_idx(zone) == ZONE_MOVABLE) 9002 continue; 9003 9004 /* 9005 * Hugepages are not in LRU lists, but they're movable. 9006 * THPs are on the LRU, but need to be counted as #small pages. 9007 * We need not scan over tail pages because we don't 9008 * handle each tail page individually in migration. 9009 */ 9010 if (PageHuge(page) || PageTransCompound(page)) { 9011 struct page *head = compound_head(page); 9012 unsigned int skip_pages; 9013 9014 if (PageHuge(page)) { 9015 if (!hugepage_migration_supported(page_hstate(head))) 9016 return page; 9017 } else if (!PageLRU(head) && !__PageMovable(head)) { 9018 return page; 9019 } 9020 9021 skip_pages = compound_nr(head) - (page - head); 9022 iter += skip_pages - 1; 9023 continue; 9024 } 9025 9026 /* 9027 * We can't use page_count without pin a page 9028 * because another CPU can free compound page. 9029 * This check already skips compound tails of THP 9030 * because their page->_refcount is zero at all time. 9031 */ 9032 if (!page_ref_count(page)) { 9033 if (PageBuddy(page)) 9034 iter += (1 << buddy_order(page)) - 1; 9035 continue; 9036 } 9037 9038 /* 9039 * The HWPoisoned page may be not in buddy system, and 9040 * page_count() is not 0. 9041 */ 9042 if ((flags & MEMORY_OFFLINE) && PageHWPoison(page)) 9043 continue; 9044 9045 /* 9046 * We treat all PageOffline() pages as movable when offlining 9047 * to give drivers a chance to decrement their reference count 9048 * in MEM_GOING_OFFLINE in order to indicate that these pages 9049 * can be offlined as there are no direct references anymore. 9050 * For actually unmovable PageOffline() where the driver does 9051 * not support this, we will fail later when trying to actually 9052 * move these pages that still have a reference count > 0. 9053 * (false negatives in this function only) 9054 */ 9055 if ((flags & MEMORY_OFFLINE) && PageOffline(page)) 9056 continue; 9057 9058 if (__PageMovable(page) || PageLRU(page)) 9059 continue; 9060 9061 /* 9062 * If there are RECLAIMABLE pages, we need to check 9063 * it. But now, memory offline itself doesn't call 9064 * shrink_node_slabs() and it still to be fixed. 9065 */ 9066 return page; 9067 } 9068 return NULL; 9069 } 9070 9071 #ifdef CONFIG_CONTIG_ALLOC 9072 static unsigned long pfn_max_align_down(unsigned long pfn) 9073 { 9074 return ALIGN_DOWN(pfn, MAX_ORDER_NR_PAGES); 9075 } 9076 9077 static unsigned long pfn_max_align_up(unsigned long pfn) 9078 { 9079 return ALIGN(pfn, MAX_ORDER_NR_PAGES); 9080 } 9081 9082 #if defined(CONFIG_DYNAMIC_DEBUG) || \ 9083 (defined(CONFIG_DYNAMIC_DEBUG_CORE) && defined(DYNAMIC_DEBUG_MODULE)) 9084 /* Usage: See admin-guide/dynamic-debug-howto.rst */ 9085 static void alloc_contig_dump_pages(struct list_head *page_list) 9086 { 9087 DEFINE_DYNAMIC_DEBUG_METADATA(descriptor, "migrate failure"); 9088 9089 if (DYNAMIC_DEBUG_BRANCH(descriptor)) { 9090 struct page *page; 9091 9092 dump_stack(); 9093 list_for_each_entry(page, page_list, lru) 9094 dump_page(page, "migration failure"); 9095 } 9096 } 9097 #else 9098 static inline void alloc_contig_dump_pages(struct list_head *page_list) 9099 { 9100 } 9101 #endif 9102 9103 /* [start, end) must belong to a single zone. */ 9104 static int __alloc_contig_migrate_range(struct compact_control *cc, 9105 unsigned long start, unsigned long end) 9106 { 9107 /* This function is based on compact_zone() from compaction.c. */ 9108 unsigned int nr_reclaimed; 9109 unsigned long pfn = start; 9110 unsigned int tries = 0; 9111 int ret = 0; 9112 struct migration_target_control mtc = { 9113 .nid = zone_to_nid(cc->zone), 9114 .gfp_mask = GFP_USER | __GFP_MOVABLE | __GFP_RETRY_MAYFAIL, 9115 }; 9116 9117 lru_cache_disable(); 9118 9119 while (pfn < end || !list_empty(&cc->migratepages)) { 9120 if (fatal_signal_pending(current)) { 9121 ret = -EINTR; 9122 break; 9123 } 9124 9125 if (list_empty(&cc->migratepages)) { 9126 cc->nr_migratepages = 0; 9127 ret = isolate_migratepages_range(cc, pfn, end); 9128 if (ret && ret != -EAGAIN) 9129 break; 9130 pfn = cc->migrate_pfn; 9131 tries = 0; 9132 } else if (++tries == 5) { 9133 ret = -EBUSY; 9134 break; 9135 } 9136 9137 nr_reclaimed = reclaim_clean_pages_from_list(cc->zone, 9138 &cc->migratepages); 9139 cc->nr_migratepages -= nr_reclaimed; 9140 9141 ret = migrate_pages(&cc->migratepages, alloc_migration_target, 9142 NULL, (unsigned long)&mtc, cc->mode, MR_CONTIG_RANGE, NULL); 9143 9144 /* 9145 * On -ENOMEM, migrate_pages() bails out right away. It is pointless 9146 * to retry again over this error, so do the same here. 9147 */ 9148 if (ret == -ENOMEM) 9149 break; 9150 } 9151 9152 lru_cache_enable(); 9153 if (ret < 0) { 9154 if (ret == -EBUSY) 9155 alloc_contig_dump_pages(&cc->migratepages); 9156 putback_movable_pages(&cc->migratepages); 9157 return ret; 9158 } 9159 return 0; 9160 } 9161 9162 /** 9163 * alloc_contig_range() -- tries to allocate given range of pages 9164 * @start: start PFN to allocate 9165 * @end: one-past-the-last PFN to allocate 9166 * @migratetype: migratetype of the underlying pageblocks (either 9167 * #MIGRATE_MOVABLE or #MIGRATE_CMA). All pageblocks 9168 * in range must have the same migratetype and it must 9169 * be either of the two. 9170 * @gfp_mask: GFP mask to use during compaction 9171 * 9172 * The PFN range does not have to be pageblock or MAX_ORDER_NR_PAGES 9173 * aligned. The PFN range must belong to a single zone. 9174 * 9175 * The first thing this routine does is attempt to MIGRATE_ISOLATE all 9176 * pageblocks in the range. Once isolated, the pageblocks should not 9177 * be modified by others. 9178 * 9179 * Return: zero on success or negative error code. On success all 9180 * pages which PFN is in [start, end) are allocated for the caller and 9181 * need to be freed with free_contig_range(). 9182 */ 9183 int alloc_contig_range(unsigned long start, unsigned long end, 9184 unsigned migratetype, gfp_t gfp_mask) 9185 { 9186 unsigned long outer_start, outer_end; 9187 unsigned int order; 9188 int ret = 0; 9189 9190 struct compact_control cc = { 9191 .nr_migratepages = 0, 9192 .order = -1, 9193 .zone = page_zone(pfn_to_page(start)), 9194 .mode = MIGRATE_SYNC, 9195 .ignore_skip_hint = true, 9196 .no_set_skip_hint = true, 9197 .gfp_mask = current_gfp_context(gfp_mask), 9198 .alloc_contig = true, 9199 }; 9200 INIT_LIST_HEAD(&cc.migratepages); 9201 9202 /* 9203 * What we do here is we mark all pageblocks in range as 9204 * MIGRATE_ISOLATE. Because pageblock and max order pages may 9205 * have different sizes, and due to the way page allocator 9206 * work, we align the range to biggest of the two pages so 9207 * that page allocator won't try to merge buddies from 9208 * different pageblocks and change MIGRATE_ISOLATE to some 9209 * other migration type. 9210 * 9211 * Once the pageblocks are marked as MIGRATE_ISOLATE, we 9212 * migrate the pages from an unaligned range (ie. pages that 9213 * we are interested in). This will put all the pages in 9214 * range back to page allocator as MIGRATE_ISOLATE. 9215 * 9216 * When this is done, we take the pages in range from page 9217 * allocator removing them from the buddy system. This way 9218 * page allocator will never consider using them. 9219 * 9220 * This lets us mark the pageblocks back as 9221 * MIGRATE_CMA/MIGRATE_MOVABLE so that free pages in the 9222 * aligned range but not in the unaligned, original range are 9223 * put back to page allocator so that buddy can use them. 9224 */ 9225 9226 ret = start_isolate_page_range(pfn_max_align_down(start), 9227 pfn_max_align_up(end), migratetype, 0); 9228 if (ret) 9229 return ret; 9230 9231 drain_all_pages(cc.zone); 9232 9233 /* 9234 * In case of -EBUSY, we'd like to know which page causes problem. 9235 * So, just fall through. test_pages_isolated() has a tracepoint 9236 * which will report the busy page. 9237 * 9238 * It is possible that busy pages could become available before 9239 * the call to test_pages_isolated, and the range will actually be 9240 * allocated. So, if we fall through be sure to clear ret so that 9241 * -EBUSY is not accidentally used or returned to caller. 9242 */ 9243 ret = __alloc_contig_migrate_range(&cc, start, end); 9244 if (ret && ret != -EBUSY) 9245 goto done; 9246 ret = 0; 9247 9248 /* 9249 * Pages from [start, end) are within a MAX_ORDER_NR_PAGES 9250 * aligned blocks that are marked as MIGRATE_ISOLATE. What's 9251 * more, all pages in [start, end) are free in page allocator. 9252 * What we are going to do is to allocate all pages from 9253 * [start, end) (that is remove them from page allocator). 9254 * 9255 * The only problem is that pages at the beginning and at the 9256 * end of interesting range may be not aligned with pages that 9257 * page allocator holds, ie. they can be part of higher order 9258 * pages. Because of this, we reserve the bigger range and 9259 * once this is done free the pages we are not interested in. 9260 * 9261 * We don't have to hold zone->lock here because the pages are 9262 * isolated thus they won't get removed from buddy. 9263 */ 9264 9265 order = 0; 9266 outer_start = start; 9267 while (!PageBuddy(pfn_to_page(outer_start))) { 9268 if (++order >= MAX_ORDER) { 9269 outer_start = start; 9270 break; 9271 } 9272 outer_start &= ~0UL << order; 9273 } 9274 9275 if (outer_start != start) { 9276 order = buddy_order(pfn_to_page(outer_start)); 9277 9278 /* 9279 * outer_start page could be small order buddy page and 9280 * it doesn't include start page. Adjust outer_start 9281 * in this case to report failed page properly 9282 * on tracepoint in test_pages_isolated() 9283 */ 9284 if (outer_start + (1UL << order) <= start) 9285 outer_start = start; 9286 } 9287 9288 /* Make sure the range is really isolated. */ 9289 if (test_pages_isolated(outer_start, end, 0)) { 9290 ret = -EBUSY; 9291 goto done; 9292 } 9293 9294 /* Grab isolated pages from freelists. */ 9295 outer_end = isolate_freepages_range(&cc, outer_start, end); 9296 if (!outer_end) { 9297 ret = -EBUSY; 9298 goto done; 9299 } 9300 9301 /* Free head and tail (if any) */ 9302 if (start != outer_start) 9303 free_contig_range(outer_start, start - outer_start); 9304 if (end != outer_end) 9305 free_contig_range(end, outer_end - end); 9306 9307 done: 9308 undo_isolate_page_range(pfn_max_align_down(start), 9309 pfn_max_align_up(end), migratetype); 9310 return ret; 9311 } 9312 EXPORT_SYMBOL(alloc_contig_range); 9313 9314 static int __alloc_contig_pages(unsigned long start_pfn, 9315 unsigned long nr_pages, gfp_t gfp_mask) 9316 { 9317 unsigned long end_pfn = start_pfn + nr_pages; 9318 9319 return alloc_contig_range(start_pfn, end_pfn, MIGRATE_MOVABLE, 9320 gfp_mask); 9321 } 9322 9323 static bool pfn_range_valid_contig(struct zone *z, unsigned long start_pfn, 9324 unsigned long nr_pages) 9325 { 9326 unsigned long i, end_pfn = start_pfn + nr_pages; 9327 struct page *page; 9328 9329 for (i = start_pfn; i < end_pfn; i++) { 9330 page = pfn_to_online_page(i); 9331 if (!page) 9332 return false; 9333 9334 if (page_zone(page) != z) 9335 return false; 9336 9337 if (PageReserved(page)) 9338 return false; 9339 } 9340 return true; 9341 } 9342 9343 static bool zone_spans_last_pfn(const struct zone *zone, 9344 unsigned long start_pfn, unsigned long nr_pages) 9345 { 9346 unsigned long last_pfn = start_pfn + nr_pages - 1; 9347 9348 return zone_spans_pfn(zone, last_pfn); 9349 } 9350 9351 /** 9352 * alloc_contig_pages() -- tries to find and allocate contiguous range of pages 9353 * @nr_pages: Number of contiguous pages to allocate 9354 * @gfp_mask: GFP mask to limit search and used during compaction 9355 * @nid: Target node 9356 * @nodemask: Mask for other possible nodes 9357 * 9358 * This routine is a wrapper around alloc_contig_range(). It scans over zones 9359 * on an applicable zonelist to find a contiguous pfn range which can then be 9360 * tried for allocation with alloc_contig_range(). This routine is intended 9361 * for allocation requests which can not be fulfilled with the buddy allocator. 9362 * 9363 * The allocated memory is always aligned to a page boundary. If nr_pages is a 9364 * power of two, then allocated range is also guaranteed to be aligned to same 9365 * nr_pages (e.g. 1GB request would be aligned to 1GB). 9366 * 9367 * Allocated pages can be freed with free_contig_range() or by manually calling 9368 * __free_page() on each allocated page. 9369 * 9370 * Return: pointer to contiguous pages on success, or NULL if not successful. 9371 */ 9372 struct page *alloc_contig_pages(unsigned long nr_pages, gfp_t gfp_mask, 9373 int nid, nodemask_t *nodemask) 9374 { 9375 unsigned long ret, pfn, flags; 9376 struct zonelist *zonelist; 9377 struct zone *zone; 9378 struct zoneref *z; 9379 9380 zonelist = node_zonelist(nid, gfp_mask); 9381 for_each_zone_zonelist_nodemask(zone, z, zonelist, 9382 gfp_zone(gfp_mask), nodemask) { 9383 spin_lock_irqsave(&zone->lock, flags); 9384 9385 pfn = ALIGN(zone->zone_start_pfn, nr_pages); 9386 while (zone_spans_last_pfn(zone, pfn, nr_pages)) { 9387 if (pfn_range_valid_contig(zone, pfn, nr_pages)) { 9388 /* 9389 * We release the zone lock here because 9390 * alloc_contig_range() will also lock the zone 9391 * at some point. If there's an allocation 9392 * spinning on this lock, it may win the race 9393 * and cause alloc_contig_range() to fail... 9394 */ 9395 spin_unlock_irqrestore(&zone->lock, flags); 9396 ret = __alloc_contig_pages(pfn, nr_pages, 9397 gfp_mask); 9398 if (!ret) 9399 return pfn_to_page(pfn); 9400 spin_lock_irqsave(&zone->lock, flags); 9401 } 9402 pfn += nr_pages; 9403 } 9404 spin_unlock_irqrestore(&zone->lock, flags); 9405 } 9406 return NULL; 9407 } 9408 #endif /* CONFIG_CONTIG_ALLOC */ 9409 9410 void free_contig_range(unsigned long pfn, unsigned long nr_pages) 9411 { 9412 unsigned long count = 0; 9413 9414 for (; nr_pages--; pfn++) { 9415 struct page *page = pfn_to_page(pfn); 9416 9417 count += page_count(page) != 1; 9418 __free_page(page); 9419 } 9420 WARN(count != 0, "%lu pages are still in use!\n", count); 9421 } 9422 EXPORT_SYMBOL(free_contig_range); 9423 9424 /* 9425 * The zone indicated has a new number of managed_pages; batch sizes and percpu 9426 * page high values need to be recalculated. 9427 */ 9428 void zone_pcp_update(struct zone *zone, int cpu_online) 9429 { 9430 mutex_lock(&pcp_batch_high_lock); 9431 zone_set_pageset_high_and_batch(zone, cpu_online); 9432 mutex_unlock(&pcp_batch_high_lock); 9433 } 9434 9435 /* 9436 * Effectively disable pcplists for the zone by setting the high limit to 0 9437 * and draining all cpus. A concurrent page freeing on another CPU that's about 9438 * to put the page on pcplist will either finish before the drain and the page 9439 * will be drained, or observe the new high limit and skip the pcplist. 9440 * 9441 * Must be paired with a call to zone_pcp_enable(). 9442 */ 9443 void zone_pcp_disable(struct zone *zone) 9444 { 9445 mutex_lock(&pcp_batch_high_lock); 9446 __zone_set_pageset_high_and_batch(zone, 0, 1); 9447 __drain_all_pages(zone, true); 9448 } 9449 9450 void zone_pcp_enable(struct zone *zone) 9451 { 9452 __zone_set_pageset_high_and_batch(zone, zone->pageset_high, zone->pageset_batch); 9453 mutex_unlock(&pcp_batch_high_lock); 9454 } 9455 9456 void zone_pcp_reset(struct zone *zone) 9457 { 9458 int cpu; 9459 struct per_cpu_zonestat *pzstats; 9460 9461 if (zone->per_cpu_pageset != &boot_pageset) { 9462 for_each_online_cpu(cpu) { 9463 pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu); 9464 drain_zonestat(zone, pzstats); 9465 } 9466 free_percpu(zone->per_cpu_pageset); 9467 free_percpu(zone->per_cpu_zonestats); 9468 zone->per_cpu_pageset = &boot_pageset; 9469 zone->per_cpu_zonestats = &boot_zonestats; 9470 } 9471 } 9472 9473 #ifdef CONFIG_MEMORY_HOTREMOVE 9474 /* 9475 * All pages in the range must be in a single zone, must not contain holes, 9476 * must span full sections, and must be isolated before calling this function. 9477 */ 9478 void __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn) 9479 { 9480 unsigned long pfn = start_pfn; 9481 struct page *page; 9482 struct zone *zone; 9483 unsigned int order; 9484 unsigned long flags; 9485 9486 offline_mem_sections(pfn, end_pfn); 9487 zone = page_zone(pfn_to_page(pfn)); 9488 spin_lock_irqsave(&zone->lock, flags); 9489 while (pfn < end_pfn) { 9490 page = pfn_to_page(pfn); 9491 /* 9492 * The HWPoisoned page may be not in buddy system, and 9493 * page_count() is not 0. 9494 */ 9495 if (unlikely(!PageBuddy(page) && PageHWPoison(page))) { 9496 pfn++; 9497 continue; 9498 } 9499 /* 9500 * At this point all remaining PageOffline() pages have a 9501 * reference count of 0 and can simply be skipped. 9502 */ 9503 if (PageOffline(page)) { 9504 BUG_ON(page_count(page)); 9505 BUG_ON(PageBuddy(page)); 9506 pfn++; 9507 continue; 9508 } 9509 9510 BUG_ON(page_count(page)); 9511 BUG_ON(!PageBuddy(page)); 9512 order = buddy_order(page); 9513 del_page_from_free_list(page, zone, order); 9514 pfn += (1 << order); 9515 } 9516 spin_unlock_irqrestore(&zone->lock, flags); 9517 } 9518 #endif 9519 9520 /* 9521 * This function returns a stable result only if called under zone lock. 9522 */ 9523 bool is_free_buddy_page(struct page *page) 9524 { 9525 unsigned long pfn = page_to_pfn(page); 9526 unsigned int order; 9527 9528 for (order = 0; order < MAX_ORDER; order++) { 9529 struct page *page_head = page - (pfn & ((1 << order) - 1)); 9530 9531 if (PageBuddy(page_head) && 9532 buddy_order_unsafe(page_head) >= order) 9533 break; 9534 } 9535 9536 return order < MAX_ORDER; 9537 } 9538 EXPORT_SYMBOL(is_free_buddy_page); 9539 9540 #ifdef CONFIG_MEMORY_FAILURE 9541 /* 9542 * Break down a higher-order page in sub-pages, and keep our target out of 9543 * buddy allocator. 9544 */ 9545 static void break_down_buddy_pages(struct zone *zone, struct page *page, 9546 struct page *target, int low, int high, 9547 int migratetype) 9548 { 9549 unsigned long size = 1 << high; 9550 struct page *current_buddy, *next_page; 9551 9552 while (high > low) { 9553 high--; 9554 size >>= 1; 9555 9556 if (target >= &page[size]) { 9557 next_page = page + size; 9558 current_buddy = page; 9559 } else { 9560 next_page = page; 9561 current_buddy = page + size; 9562 } 9563 9564 if (set_page_guard(zone, current_buddy, high, migratetype)) 9565 continue; 9566 9567 if (current_buddy != target) { 9568 add_to_free_list(current_buddy, zone, high, migratetype); 9569 set_buddy_order(current_buddy, high); 9570 page = next_page; 9571 } 9572 } 9573 } 9574 9575 /* 9576 * Take a page that will be marked as poisoned off the buddy allocator. 9577 */ 9578 bool take_page_off_buddy(struct page *page) 9579 { 9580 struct zone *zone = page_zone(page); 9581 unsigned long pfn = page_to_pfn(page); 9582 unsigned long flags; 9583 unsigned int order; 9584 bool ret = false; 9585 9586 spin_lock_irqsave(&zone->lock, flags); 9587 for (order = 0; order < MAX_ORDER; order++) { 9588 struct page *page_head = page - (pfn & ((1 << order) - 1)); 9589 int page_order = buddy_order(page_head); 9590 9591 if (PageBuddy(page_head) && page_order >= order) { 9592 unsigned long pfn_head = page_to_pfn(page_head); 9593 int migratetype = get_pfnblock_migratetype(page_head, 9594 pfn_head); 9595 9596 del_page_from_free_list(page_head, zone, page_order); 9597 break_down_buddy_pages(zone, page_head, page, 0, 9598 page_order, migratetype); 9599 SetPageHWPoisonTakenOff(page); 9600 if (!is_migrate_isolate(migratetype)) 9601 __mod_zone_freepage_state(zone, -1, migratetype); 9602 ret = true; 9603 break; 9604 } 9605 if (page_count(page_head) > 0) 9606 break; 9607 } 9608 spin_unlock_irqrestore(&zone->lock, flags); 9609 return ret; 9610 } 9611 9612 /* 9613 * Cancel takeoff done by take_page_off_buddy(). 9614 */ 9615 bool put_page_back_buddy(struct page *page) 9616 { 9617 struct zone *zone = page_zone(page); 9618 unsigned long pfn = page_to_pfn(page); 9619 unsigned long flags; 9620 int migratetype = get_pfnblock_migratetype(page, pfn); 9621 bool ret = false; 9622 9623 spin_lock_irqsave(&zone->lock, flags); 9624 if (put_page_testzero(page)) { 9625 ClearPageHWPoisonTakenOff(page); 9626 __free_one_page(page, pfn, zone, 0, migratetype, FPI_NONE); 9627 if (TestClearPageHWPoison(page)) { 9628 num_poisoned_pages_dec(); 9629 ret = true; 9630 } 9631 } 9632 spin_unlock_irqrestore(&zone->lock, flags); 9633 9634 return ret; 9635 } 9636 #endif 9637 9638 #ifdef CONFIG_ZONE_DMA 9639 bool has_managed_dma(void) 9640 { 9641 struct pglist_data *pgdat; 9642 9643 for_each_online_pgdat(pgdat) { 9644 struct zone *zone = &pgdat->node_zones[ZONE_DMA]; 9645 9646 if (managed_zone(zone)) 9647 return true; 9648 } 9649 return false; 9650 } 9651 #endif /* CONFIG_ZONE_DMA */ 9652